first commit
This commit is contained in:
commit
942b698dd0
GPG Key ID:
1688DA83D222D0B5
|
|
@ -0,0 +1,25 @@
|
|||
module.exports = {
|
||||
parser: '@typescript-eslint/parser',
|
||||
parserOptions: {
|
||||
project: 'tsconfig.json',
|
||||
tsconfigRootDir : __dirname,
|
||||
sourceType: 'module',
|
||||
},
|
||||
plugins: ['@typescript-eslint/eslint-plugin'],
|
||||
extends: [
|
||||
'plugin:@typescript-eslint/recommended',
|
||||
'plugin:prettier/recommended',
|
||||
],
|
||||
root: true,
|
||||
env: {
|
||||
node: true,
|
||||
jest: true,
|
||||
},
|
||||
ignorePatterns: ['.eslintrc.js'],
|
||||
rules: {
|
||||
'@typescript-eslint/interface-name-prefix': 'off',
|
||||
'@typescript-eslint/explicit-function-return-type': 'off',
|
||||
'@typescript-eslint/explicit-module-boundary-types': 'off',
|
||||
'@typescript-eslint/no-explicit-any': 'off',
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
# compiled output
|
||||
/dist
|
||||
/node_modules
|
||||
.gntmp*
|
||||
/geobase
|
||||
*.db
|
||||
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
pnpm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
|
||||
# Tests
|
||||
/coverage
|
||||
/.nyc_output
|
||||
|
||||
# IDEs and editors
|
||||
/.idea
|
||||
.project
|
||||
.classpath
|
||||
.c9/
|
||||
*.launch
|
||||
.settings/
|
||||
*.sublime-workspace
|
||||
|
||||
# IDE - VSCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all"
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"editor.formatOnSave": true
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
<p align="center">
|
||||
<a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo-small.svg" width="200" alt="Nest Logo" /></a>
|
||||
</p>
|
||||
|
||||
[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
|
||||
[circleci-url]: https://circleci.com/gh/nestjs/nest
|
||||
|
||||
<p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
|
||||
<p align="center">
|
||||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
|
||||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
|
||||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
|
||||
<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
|
||||
<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
|
||||
<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
|
||||
<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
|
||||
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
|
||||
<a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
|
||||
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
|
||||
<a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
|
||||
</p>
|
||||
<!--[](https://opencollective.com/nest#backer)
|
||||
[](https://opencollective.com/nest#sponsor)-->
|
||||
|
||||
## Description
|
||||
|
||||
[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
$ npm install
|
||||
```
|
||||
|
||||
## Running the app
|
||||
|
||||
```
|
||||
$ docker run --rm --name geobase-db -p 22336:3306 -v $PWD/geobase:/var/lib/mysql --env MARIADB_USER=geobase --env MARIADB_PASSWORD=geobase --env MARIADB_ROOT_PASSWORD=geobase --env MARIADB_DATABASE=geobase mariadb:latest
|
||||
```
|
||||
|
||||
```bash
|
||||
# development
|
||||
$ npm run start
|
||||
|
||||
# watch mode
|
||||
$ npm run start:dev
|
||||
|
||||
# production mode
|
||||
$ npm run start:prod
|
||||
```
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
# unit tests
|
||||
$ npm run test
|
||||
|
||||
# e2e tests
|
||||
$ npm run test:e2e
|
||||
|
||||
# test coverage
|
||||
$ npm run test:cov
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
|
||||
|
||||
## Stay in touch
|
||||
|
||||
- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
|
||||
- Website - [https://nestjs.com](https://nestjs.com/)
|
||||
- Twitter - [@nestframework](https://twitter.com/nestframework)
|
||||
|
||||
## License
|
||||
|
||||
Nest is [MIT licensed](LICENSE).
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import fs from 'fs';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { join } from 'path';
|
||||
import unzipper from 'unzipper';
|
||||
|
||||
const GEONAMES_DUMP =
|
||||
'https://download.geonames.org/export/dump/allCountries.zip';
|
||||
|
||||
async function workDirectory() {
|
||||
const path = await fs.promises.mkdtemp(join(process.cwd(), '.gntmp-'));
|
||||
return {
|
||||
path,
|
||||
remove: async () => {
|
||||
await fs.promises.rm(path, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadGeodump({ path }) {
|
||||
const outfile = join(path, 'out.txt');
|
||||
const httpStream = await fetch(GEONAMES_DUMP);
|
||||
pipeline(httpStream.body, unzipper.Extract({ path }));
|
||||
return outfile;
|
||||
}
|
||||
|
||||
async function test() {
|
||||
const wd = await workDirectory();
|
||||
await downloadGeodump(wd);
|
||||
}
|
||||
|
||||
test().catch(console.error);
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"$schema": "https://json.schemastore.org/nest-cli",
|
||||
"collection": "@nestjs/schematics",
|
||||
"sourceRoot": "src"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,79 @@
|
|||
{
|
||||
"name": "evert-earth-utils",
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"author": "",
|
||||
"private": true,
|
||||
"license": "UNLICENSED",
|
||||
"scripts": {
|
||||
"prebuild": "rimraf dist",
|
||||
"build": "nest build",
|
||||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
||||
"start": "nest start",
|
||||
"start:dev": "nest start --watch",
|
||||
"start:debug": "nest start --debug --watch",
|
||||
"start:prod": "node dist/main",
|
||||
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
||||
"test": "jest",
|
||||
"test:watch": "jest --watch",
|
||||
"test:cov": "jest --coverage",
|
||||
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
|
||||
"test:e2e": "jest --config ./test/jest-e2e.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@nestjs/common": "^9.0.0",
|
||||
"@nestjs/core": "^9.0.0",
|
||||
"@nestjs/platform-express": "^9.0.0",
|
||||
"@nestjs/typeorm": "^9.0.1",
|
||||
"mysql2": "^2.3.3",
|
||||
"nestjs-command": "^3.1.2",
|
||||
"reflect-metadata": "^0.1.13",
|
||||
"rimraf": "^3.0.2",
|
||||
"rxjs": "^7.2.0",
|
||||
"sqlite": "^4.1.2",
|
||||
"sqlite3": "^5.1.4",
|
||||
"typeorm": "^0.3.11",
|
||||
"unzipper": "^0.10.11"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nestjs/cli": "^9.0.0",
|
||||
"@nestjs/schematics": "^9.0.0",
|
||||
"@nestjs/testing": "^9.0.0",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/jest": "28.1.4",
|
||||
"@types/node": "^16.0.0",
|
||||
"@types/supertest": "^2.0.11",
|
||||
"@types/unzipper": "^0.10.5",
|
||||
"@typescript-eslint/eslint-plugin": "^5.0.0",
|
||||
"@typescript-eslint/parser": "^5.0.0",
|
||||
"eslint": "^8.0.1",
|
||||
"eslint-config-prettier": "^8.3.0",
|
||||
"eslint-plugin-prettier": "^4.0.0",
|
||||
"jest": "28.1.2",
|
||||
"prettier": "^2.3.2",
|
||||
"source-map-support": "^0.5.20",
|
||||
"supertest": "^6.1.3",
|
||||
"ts-jest": "28.0.5",
|
||||
"ts-loader": "^9.2.3",
|
||||
"ts-node": "^10.0.0",
|
||||
"tsconfig-paths": "4.0.0",
|
||||
"typescript": "^4.3.5"
|
||||
},
|
||||
"jest": {
|
||||
"moduleFileExtensions": [
|
||||
"js",
|
||||
"json",
|
||||
"ts"
|
||||
],
|
||||
"rootDir": "src",
|
||||
"testRegex": ".*\\.spec\\.ts$",
|
||||
"transform": {
|
||||
"^.+\\.(t|j)s$": "ts-jest"
|
||||
},
|
||||
"collectCoverageFrom": [
|
||||
"**/*.(t|j)s"
|
||||
],
|
||||
"coverageDirectory": "../coverage",
|
||||
"testEnvironment": "node"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { AppController } from './app.controller';
|
||||
import { AppService } from './app.service';
|
||||
|
||||
describe('AppController', () => {
|
||||
let appController: AppController;
|
||||
|
||||
beforeEach(async () => {
|
||||
const app: TestingModule = await Test.createTestingModule({
|
||||
controllers: [AppController],
|
||||
providers: [AppService],
|
||||
}).compile();
|
||||
|
||||
appController = app.get<AppController>(AppController);
|
||||
});
|
||||
|
||||
describe('root', () => {
|
||||
it('should return "Hello World!"', () => {
|
||||
expect(appController.getHello()).toBe('Hello World!');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
import { Controller, Get } from '@nestjs/common';
|
||||
import { AppService } from './app.service';
|
||||
|
||||
@Controller()
|
||||
export class AppController {
|
||||
constructor(private readonly appService: AppService) {}
|
||||
|
||||
@Get()
|
||||
getHello(): string {
|
||||
return this.appService.getHello();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { AppController } from './app.controller';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { AppService } from './app.service';
|
||||
import { CommandModule } from 'nestjs-command';
|
||||
import { GeonamesModule } from './modules/geonames/geonames.module';
|
||||
import { Geoname } from './modules/geonames/geonames.entity';
|
||||
import { Country } from './modules/countries/countries.entity';
|
||||
import { CountriesModule } from './modules/countries/countries.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forRoot({
|
||||
type: 'mysql',
|
||||
host: 'localhost',
|
||||
port: 22336,
|
||||
username: 'geobase',
|
||||
password: 'geobase',
|
||||
database: 'geobase',
|
||||
name: 'geobase',
|
||||
entities: [Geoname, Country],
|
||||
synchronize: true,
|
||||
}),
|
||||
CommandModule,
|
||||
GeonamesModule,
|
||||
CountriesModule,
|
||||
],
|
||||
controllers: [AppController],
|
||||
providers: [AppService],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class AppService {
|
||||
getHello(): string {
|
||||
return 'Hello World!';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
import { NestFactory } from '@nestjs/core';
|
||||
import { CommandModule, CommandService } from 'nestjs-command';
|
||||
import { AppModule } from './app.module';
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await NestFactory.createApplicationContext(AppModule);
|
||||
|
||||
try {
|
||||
await app.select(CommandModule).get(CommandService).exec();
|
||||
await app.close();
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
await app.close();
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
bootstrap();
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import { NestFactory } from '@nestjs/core';
|
||||
import { AppModule } from './app.module';
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await NestFactory.create(AppModule);
|
||||
await app.listen(3000);
|
||||
}
|
||||
bootstrap();
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import { Injectable } from '@nestjs/common';
|
||||
import { Command } from 'nestjs-command';
|
||||
import { CountriesService } from './countries.service';
|
||||
|
||||
@Injectable()
|
||||
export class CountriesCommand {
|
||||
constructor(private readonly service: CountriesService) {}
|
||||
|
||||
@Command({
|
||||
command: 'countries:import',
|
||||
describe: 'Import Countries database',
|
||||
})
|
||||
async import() {
|
||||
await this.service.pullCountries();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
import { Controller, Get, Param, Query } from '@nestjs/common';
|
||||
import { CountriesQueryDto } from './countries.interfaces';
|
||||
import { CountriesService } from './countries.service';
|
||||
|
||||
@Controller({
|
||||
path: '/countries',
|
||||
})
|
||||
export class CountriesController {
|
||||
constructor(private readonly service: CountriesService) {}
|
||||
|
||||
@Get()
|
||||
async getAllCountries(@Query() { q, fields }: CountriesQueryDto) {
|
||||
return this.service.search(q, fields);
|
||||
}
|
||||
|
||||
@Get(':iso')
|
||||
async getByISO(
|
||||
@Param('iso') iso: string,
|
||||
@Query() { fields }: CountriesQueryDto,
|
||||
) {
|
||||
return this.service.getByISO(iso.toUpperCase(), fields);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import { Column, Entity, Index, PrimaryColumn } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class Country {
|
||||
@PrimaryColumn({ length: 2 })
|
||||
iso: string;
|
||||
|
||||
@Index()
|
||||
@Column({ length: 3, nullable: true })
|
||||
iso3: string;
|
||||
|
||||
@Column({ length: 3, nullable: true })
|
||||
isoNumeric: string;
|
||||
|
||||
@Column({ length: 2, nullable: true })
|
||||
fips: string;
|
||||
|
||||
@Index()
|
||||
@Column()
|
||||
country: string;
|
||||
|
||||
@Index()
|
||||
@Column()
|
||||
capital: string;
|
||||
|
||||
@Column()
|
||||
area: number;
|
||||
|
||||
@Column()
|
||||
population: number;
|
||||
|
||||
@Column({ nullable: true })
|
||||
continent: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
tld: string;
|
||||
|
||||
@Index()
|
||||
@Column({ nullable: true })
|
||||
currencyCode: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
currencyName: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
phone: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
postalCodeFormat: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
postalCodeRegex: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
languages: string;
|
||||
|
||||
@Index()
|
||||
@Column()
|
||||
geonameid: number;
|
||||
|
||||
@Column({ nullable: true })
|
||||
neighbours: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
equivalentFipsCode: string;
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
export interface CountriesQueryDto {
|
||||
fields?: string[];
|
||||
q?: string;
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { CommandModule } from 'nestjs-command';
|
||||
import { CountriesCommand } from './countries.command';
|
||||
import { CountriesController } from './countries.controller';
|
||||
import { Country } from './countries.entity';
|
||||
import { CountriesService } from './countries.service';
|
||||
|
||||
@Module({
|
||||
imports: [CommandModule, TypeOrmModule.forFeature([Country], 'geobase')],
|
||||
controllers: [CountriesController],
|
||||
providers: [CountriesService, CountriesCommand],
|
||||
exports: [CountriesService, TypeOrmModule],
|
||||
})
|
||||
export class CountriesModule {}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { ILike, Repository } from 'typeorm';
|
||||
import { Country } from './countries.entity';
|
||||
import { intOrNull } from 'src/utils/int-or-null';
|
||||
|
||||
const COUNTRIES_URL =
|
||||
'https://download.geonames.org/export/dump/countryInfo.txt';
|
||||
|
||||
const ACCEPT_FIELDS = [
|
||||
'iso',
|
||||
'iso3',
|
||||
'isoNumeric',
|
||||
'fips',
|
||||
'country',
|
||||
'capital',
|
||||
'area',
|
||||
'population',
|
||||
'continent',
|
||||
'tld',
|
||||
'currencyCode',
|
||||
'currencyName',
|
||||
'phone',
|
||||
'postalCodeFormat',
|
||||
'postalCodeRegex',
|
||||
'languages',
|
||||
'geonameid',
|
||||
'neighbours',
|
||||
'equivalentFipsCode',
|
||||
];
|
||||
@Injectable()
|
||||
export class CountriesService {
|
||||
constructor(
|
||||
@InjectRepository(Country, 'geobase')
|
||||
private countryRepository: Repository<Country>,
|
||||
) {}
|
||||
|
||||
private mapAllowedQuery(select: string[]) {
|
||||
return (Array.isArray(select) ? select : [select]).filter((field) =>
|
||||
ACCEPT_FIELDS.includes(field),
|
||||
) as unknown as (keyof Country)[];
|
||||
}
|
||||
|
||||
async pullCountries() {
|
||||
const countryList = await fetch(COUNTRIES_URL).then((x) => x.text());
|
||||
const entries = countryList
|
||||
.split('\n')
|
||||
.filter((line) => line && !line.startsWith('#'))
|
||||
.map((line) => line.replace('\r', '').split('\t'))
|
||||
.map((entry) => {
|
||||
const country = new Country();
|
||||
Object.assign(country, {
|
||||
iso: entry[0],
|
||||
iso3: entry[1],
|
||||
isoNumeric: entry[2],
|
||||
fips: entry[3],
|
||||
country: entry[4],
|
||||
capital: entry[5],
|
||||
area: parseFloat(entry[6] || '0'),
|
||||
population: intOrNull(entry[7]),
|
||||
continent: entry[8],
|
||||
tld: entry[9],
|
||||
currencyCode: entry[10],
|
||||
currencyName: entry[11],
|
||||
phone: entry[12],
|
||||
postalCodeFormat: entry[13],
|
||||
postalCodeRegex: entry[14],
|
||||
languages: entry[15],
|
||||
geonameid: intOrNull(entry[16]),
|
||||
neighbours: entry[17],
|
||||
equivalentFipsCode: entry[18],
|
||||
});
|
||||
return country;
|
||||
});
|
||||
await this.countryRepository.save(entries);
|
||||
}
|
||||
|
||||
async getAll(fields = ACCEPT_FIELDS) {
|
||||
const select = this.mapAllowedQuery(fields);
|
||||
|
||||
return this.countryRepository.find({
|
||||
select,
|
||||
});
|
||||
}
|
||||
|
||||
async search(query?: string, fields = ACCEPT_FIELDS) {
|
||||
const select = this.mapAllowedQuery(fields);
|
||||
const filter = {};
|
||||
|
||||
if (query) {
|
||||
filter['country'] = ILike(`%${query}%`);
|
||||
}
|
||||
|
||||
return this.countryRepository.find({
|
||||
where: filter,
|
||||
select,
|
||||
});
|
||||
}
|
||||
|
||||
async getByISO(iso: string, fields = ACCEPT_FIELDS) {
|
||||
const select = this.mapAllowedQuery(fields);
|
||||
|
||||
const find = await this.countryRepository.findOne({
|
||||
where: {
|
||||
[iso.length === 2 ? 'iso' : 'iso3']: iso,
|
||||
},
|
||||
select,
|
||||
});
|
||||
|
||||
if (!find) {
|
||||
throw new NotFoundException();
|
||||
}
|
||||
|
||||
return find;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,16 @@
|
|||
import { Injectable } from '@nestjs/common';
|
||||
import { Command } from 'nestjs-command';
|
||||
import { GeonamesService } from './geonames.service';
|
||||
|
||||
@Injectable()
|
||||
export class GeonamesCommand {
|
||||
constructor(private readonly service: GeonamesService) {}
|
||||
|
||||
@Command({
|
||||
command: 'geonames:import',
|
||||
describe: 'Import Geonames database',
|
||||
})
|
||||
async import() {
|
||||
await this.service.runUpdateCycle();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import { Controller, Get, Query } from '@nestjs/common';
|
||||
import { GeonameQuery } from './geonames.interfaces';
|
||||
import { GeonamesService } from './geonames.service';
|
||||
|
||||
@Controller({
|
||||
path: 'geonames',
|
||||
})
|
||||
export class GeonamesController {
|
||||
constructor(private readonly service: GeonamesService) {}
|
||||
|
||||
@Get()
|
||||
async search(@Query() query: GeonameQuery) {
|
||||
return this.service.search(query);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import { Entity, Column, Index, PrimaryGeneratedColumn } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class Geoname {
|
||||
@PrimaryGeneratedColumn()
|
||||
geonameid: number;
|
||||
|
||||
@Index()
|
||||
@Column({ length: 200 })
|
||||
name: string;
|
||||
|
||||
@Column({ nullable: true, length: 200 })
|
||||
asciiname: string;
|
||||
|
||||
@Column({ nullable: true, type: 'text' })
|
||||
alternatenames: string;
|
||||
|
||||
@Column({ nullable: true, type: 'real' })
|
||||
latitude: number;
|
||||
|
||||
@Column({ nullable: true, type: 'real' })
|
||||
longitude: number;
|
||||
|
||||
// http://www.geonames.org/export/codes.html
|
||||
@Index()
|
||||
@Column({ type: 'char', length: 1 })
|
||||
featureclass: string;
|
||||
|
||||
// http://www.geonames.org/export/codes.html
|
||||
@Index()
|
||||
@Column({ length: 10 })
|
||||
featurecode: string;
|
||||
|
||||
@Index()
|
||||
@Column({ length: 2 })
|
||||
countrycode: string;
|
||||
|
||||
@Column({ nullable: true, length: 200 })
|
||||
cc2: string;
|
||||
|
||||
@Column({ nullable: true, length: 20 })
|
||||
admin1code: string;
|
||||
|
||||
@Column({ nullable: true, length: 80 })
|
||||
admin2code: string;
|
||||
|
||||
@Column({ nullable: true, length: 20 })
|
||||
admin3code: string;
|
||||
|
||||
@Column({ nullable: true, length: 20 })
|
||||
admin4code: string;
|
||||
|
||||
@Column({ nullable: true, type: 'bigint' })
|
||||
population: number;
|
||||
|
||||
@Column({ nullable: true })
|
||||
elevation: number;
|
||||
|
||||
@Column({ nullable: true })
|
||||
dem: number;
|
||||
|
||||
@Index()
|
||||
@Column({ nullable: true })
|
||||
timezone: string;
|
||||
|
||||
@Column({ type: 'date' })
|
||||
moddate: string;
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import { Geoname } from './geonames.entity';
|
||||
|
||||
export interface WorkDirectory {
|
||||
path: string;
|
||||
remove(): Promise<void>;
|
||||
}
|
||||
|
||||
export type GeonameQuery = Record<keyof Geoname, unknown> & {
|
||||
fields?: (keyof Geoname)[];
|
||||
limit?: string;
|
||||
offset?: string;
|
||||
q?: string;
|
||||
};
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { CommandModule } from 'nestjs-command';
|
||||
import { GeonamesCommand } from './geonames.command';
|
||||
import { GeonamesController } from './geonames.controller';
|
||||
import { Geoname } from './geonames.entity';
|
||||
import { GeonamesService } from './geonames.service';
|
||||
|
||||
@Module({
|
||||
imports: [CommandModule, TypeOrmModule.forFeature([Geoname], 'geobase')],
|
||||
controllers: [GeonamesController],
|
||||
providers: [GeonamesService, GeonamesCommand],
|
||||
exports: [GeonamesService],
|
||||
})
|
||||
export class GeonamesModule {}
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
|
||||
import { DataSource, ILike, In, Repository } from 'typeorm';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { join } from 'path';
|
||||
import { GeonameQuery, WorkDirectory } from './geonames.interfaces';
|
||||
import { createInterface } from 'readline';
|
||||
import { Geoname } from './geonames.entity';
|
||||
import { intOrNull } from 'src/utils/int-or-null';
|
||||
import * as fs from 'fs';
|
||||
import * as unzipper from 'unzipper';
|
||||
|
||||
const GEONAMES_DUMP =
|
||||
'https://download.geonames.org/export/dump/allCountries.zip';
|
||||
|
||||
const context = 'GeonamesImport';
|
||||
|
||||
const ACCEPT_FIELDS = [
|
||||
'geonameid',
|
||||
'name',
|
||||
'asciiname',
|
||||
'alternatenames',
|
||||
'latitude',
|
||||
'longitude',
|
||||
'featureclass',
|
||||
'featurecode',
|
||||
'countrycode',
|
||||
'cc2',
|
||||
'admin1code',
|
||||
'admin2code',
|
||||
'admin3code',
|
||||
'admin4code',
|
||||
'population',
|
||||
'elevation',
|
||||
'dem',
|
||||
'timezone',
|
||||
'moddate',
|
||||
];
|
||||
|
||||
type ReduceType = Partial<Record<keyof Geoname, unknown>>;
|
||||
|
||||
@Injectable()
|
||||
export class GeonamesService {
|
||||
constructor(
|
||||
@InjectDataSource('geobase')
|
||||
private dataSource: DataSource,
|
||||
@InjectRepository(Geoname, 'geobase')
|
||||
private geonameRepository: Repository<Geoname>,
|
||||
) {}
|
||||
|
||||
private mapAllowedQuery(select: string[]) {
|
||||
return (Array.isArray(select) ? select : [select]).filter((field) =>
|
||||
ACCEPT_FIELDS.includes(field),
|
||||
) as unknown as (keyof Geoname)[];
|
||||
}
|
||||
|
||||
async search(params: GeonameQuery) {
|
||||
const select = this.mapAllowedQuery(params.fields || ACCEPT_FIELDS);
|
||||
let where = Object.keys(params)
|
||||
.filter((key) => !['fields', 'limit', 'q', 'offset'].includes(key))
|
||||
.reduce<ReduceType>(
|
||||
(obj, key) =>
|
||||
ACCEPT_FIELDS.includes(key)
|
||||
? {
|
||||
...obj,
|
||||
[key]: Array.isArray(params[key])
|
||||
? In(params[key])
|
||||
: params[key],
|
||||
}
|
||||
: obj,
|
||||
{},
|
||||
) as ReduceType | ReduceType[];
|
||||
|
||||
if (params.q) {
|
||||
let searchTerm = params.q;
|
||||
if (searchTerm.startsWith('ext:')) {
|
||||
searchTerm = searchTerm.substring(4);
|
||||
where = ['name', 'asciiname', 'alternatenames'].map((field) => ({
|
||||
...(where as ReduceType),
|
||||
[field as keyof Geoname]: ILike(`%${searchTerm}%`),
|
||||
}));
|
||||
} else {
|
||||
where['name'] = ILike(`%${searchTerm}%`);
|
||||
}
|
||||
}
|
||||
|
||||
const take = Math.max(
|
||||
Math.min(intOrNull(params.limit as string) || 50, 1000),
|
||||
1,
|
||||
);
|
||||
|
||||
const skip = intOrNull(params.offset as string) || 0;
|
||||
return this.geonameRepository.find({
|
||||
select,
|
||||
where: where as unknown,
|
||||
skip,
|
||||
take,
|
||||
});
|
||||
}
|
||||
|
||||
async workDirectory(): Promise<WorkDirectory> {
|
||||
const path = await fs.promises.mkdtemp(join(process.cwd(), '.gntmp-'));
|
||||
return {
|
||||
path,
|
||||
remove: async () => {
|
||||
await fs.promises.rm(path, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async downloadGeodump({ path }: WorkDirectory): Promise<string> {
|
||||
const outfile = join(path, 'out.txt');
|
||||
const output = fs.createWriteStream(outfile);
|
||||
const httpStream = await fetch(GEONAMES_DUMP);
|
||||
|
||||
await pipeline(
|
||||
httpStream.body as unknown as NodeJS.ReadableStream,
|
||||
unzipper.Parse().on('entry', (entry) => {
|
||||
if (entry.path === 'allCountries.txt') {
|
||||
entry.pipe(output);
|
||||
} else {
|
||||
entry.autodrain();
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
return outfile;
|
||||
}
|
||||
|
||||
async parseGeodump(path: string) {
|
||||
const read = fs.createReadStream(path);
|
||||
const rl = createInterface({
|
||||
terminal: false,
|
||||
input: read,
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
const queryRunner = this.dataSource.createQueryRunner();
|
||||
await queryRunner.connect();
|
||||
await queryRunner.startTransaction();
|
||||
|
||||
let entities = 0;
|
||||
let totalBatches = 0;
|
||||
let totalEntities = 0;
|
||||
|
||||
for await (const line of rl) {
|
||||
const split = line.split('\t');
|
||||
const model = new Geoname();
|
||||
Object.assign(model, {
|
||||
geonameid: parseInt(split[0], 10),
|
||||
name: split[1],
|
||||
asciiname: split[2],
|
||||
alternatenames: split[3],
|
||||
latitude: parseFloat(split[4]) || null,
|
||||
longitude: parseFloat(split[5]) || null,
|
||||
featureclass: split[6],
|
||||
featurecode: split[7],
|
||||
countrycode: split[8],
|
||||
cc2: split[9],
|
||||
admin1code: split[10] || null,
|
||||
admin2code: split[11] || null,
|
||||
admin3code: split[12] || null,
|
||||
admin4code: split[13] || null,
|
||||
population: intOrNull(split[14]),
|
||||
elevation: intOrNull(split[15]),
|
||||
dem: intOrNull(split[16]),
|
||||
timezone: split[17],
|
||||
moddate: split[18],
|
||||
});
|
||||
|
||||
await queryRunner.manager.save(model);
|
||||
entities += 1;
|
||||
totalEntities += 1;
|
||||
|
||||
if (entities >= 100) {
|
||||
totalBatches += 1;
|
||||
try {
|
||||
await queryRunner.commitTransaction();
|
||||
} catch (err) {
|
||||
Logger.error(`Some fields failed to insert: ${err}`, context);
|
||||
await queryRunner.rollbackTransaction();
|
||||
}
|
||||
|
||||
if (totalBatches % 10 === 0) {
|
||||
Logger.log(
|
||||
`Batch ${totalBatches} committed, ${totalEntities} entities so far`,
|
||||
context,
|
||||
);
|
||||
}
|
||||
await queryRunner.startTransaction();
|
||||
entities = 0;
|
||||
}
|
||||
}
|
||||
await queryRunner.commitTransaction();
|
||||
await queryRunner.release();
|
||||
}
|
||||
|
||||
async runUpdateCycle() {
|
||||
Logger.log('Starting geonames importer', context);
|
||||
const workDirectory = await this.workDirectory();
|
||||
|
||||
try {
|
||||
Logger.log('Downloading dump...', context);
|
||||
// const file = await this.downloadGeodump(workDirectory);
|
||||
|
||||
Logger.log('Creating database...', context);
|
||||
await this.parseGeodump(
|
||||
'/home/evert/Projects/evert-earth-utils/.gntmp-sof1ES/out.txt',
|
||||
); //file);
|
||||
} catch (e) {
|
||||
await workDirectory.remove();
|
||||
throw e;
|
||||
}
|
||||
|
||||
Logger.log('Cleaning up', context);
|
||||
await workDirectory.remove();
|
||||
|
||||
Logger.log('Done!', context);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
export const intOrNull = (ref: string) => {
|
||||
if (!ref) return null;
|
||||
const parse = parseInt(ref, 10);
|
||||
if (isNaN(parse)) return null;
|
||||
return parse;
|
||||
};
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import * as request from 'supertest';
|
||||
import { AppModule } from './../src/app.module';
|
||||
|
||||
describe('AppController (e2e)', () => {
|
||||
let app: INestApplication;
|
||||
|
||||
beforeEach(async () => {
|
||||
const moduleFixture: TestingModule = await Test.createTestingModule({
|
||||
imports: [AppModule],
|
||||
}).compile();
|
||||
|
||||
app = moduleFixture.createNestApplication();
|
||||
await app.init();
|
||||
});
|
||||
|
||||
it('/ (GET)', () => {
|
||||
return request(app.getHttpServer())
|
||||
.get('/')
|
||||
.expect(200)
|
||||
.expect('Hello World!');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"moduleFileExtensions": ["js", "json", "ts"],
|
||||
"rootDir": ".",
|
||||
"testEnvironment": "node",
|
||||
"testRegex": ".e2e-spec.ts$",
|
||||
"transform": {
|
||||
"^.+\\.(t|j)s$": "ts-jest"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"declaration": true,
|
||||
"removeComments": true,
|
||||
"emitDecoratorMetadata": true,
|
||||
"experimentalDecorators": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"target": "es2017",
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"baseUrl": "./",
|
||||
"incremental": true,
|
||||
"skipLibCheck": true,
|
||||
"strictNullChecks": false,
|
||||
"noImplicitAny": false,
|
||||
"strictBindCallApply": false,
|
||||
"forceConsistentCasingInFileNames": false,
|
||||
"noFallthroughCasesInSwitch": false
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue