From c227dc86bb9ee8047b888c109a0a297605f44e16 Mon Sep 17 00:00:00 2001 From: Ali BARIN Date: Fri, 8 Apr 2022 11:36:35 +0200 Subject: [PATCH] feat(cli): create DB in start command if not exists (#285) --- packages/backend/bin/database/create.ts | 41 +-------- packages/backend/bin/database/drop.ts | 28 +----- packages/backend/bin/database/seed-user.ts | 14 +-- packages/backend/bin/database/utils.ts | 100 +++++++++++++++++++++ packages/backend/database.d.ts | 2 + packages/backend/database.js | 3 + packages/backend/logger.d.ts | 1 + packages/backend/logger.js | 2 + packages/backend/package.json | 15 +++- packages/backend/server.d.ts | 1 + packages/backend/server.js | 2 + packages/backend/src/config/database.ts | 6 +- packages/backend/src/config/orm.ts | 4 +- packages/backend/src/helpers/logger.ts | 22 ++--- packages/backend/src/models/connection.ts | 2 +- packages/backend/tsconfig.json | 7 +- packages/cli/src/commands/start.ts | 41 ++++++++- packages/cli/tsconfig.json | 1 - yarn.lock | 45 +++++++--- 19 files changed, 211 insertions(+), 126 deletions(-) create mode 100644 packages/backend/bin/database/utils.ts create mode 100644 packages/backend/database.d.ts create mode 100644 packages/backend/database.js create mode 100644 packages/backend/logger.d.ts create mode 100644 packages/backend/logger.js create mode 100644 packages/backend/server.d.ts create mode 100644 packages/backend/server.js diff --git a/packages/backend/bin/database/create.ts b/packages/backend/bin/database/create.ts index 20585ff3..9b13967d 100644 --- a/packages/backend/bin/database/create.ts +++ b/packages/backend/bin/database/create.ts @@ -1,42 +1,3 @@ -import appConfig from '../../src/config/app'; -import logger from '../../src/helpers/logger'; -import client from './client'; - -const createDatabaseAndUser = async () => { - if(appConfig.appEnv !== 'development' && appConfig.appEnv !== 'test') { - const errorMessage = 'Database creation can be used only with development or test environments!' - logger.error(errorMessage) - - return; - } - - await client.connect(); - await createDatabase(); - await createDatabaseUser(); - await grantPrivileges(); - - await client.end(); -} - -const createDatabase = async () => { - await client.query(`CREATE DATABASE ${appConfig.postgresDatabase}`); - logger.info(`Database: ${appConfig.postgresDatabase} created!`); -} - -const createDatabaseUser = async () => { - await client.query(`CREATE USER ${appConfig.postgresUsername}`); - logger.info(`Database User: ${appConfig.postgresUsername} created!`); -} - -const grantPrivileges = async () => { - await client.query( - `GRANT ALL PRIVILEGES ON DATABASE ${appConfig.postgresDatabase} TO ${appConfig.postgresUsername};` - ); - - logger.info( - `${appConfig.postgresUsername} has granted all privileges on ${appConfig.postgresDatabase}!` - ); - -} +import { createDatabaseAndUser } from './utils'; createDatabaseAndUser(); diff --git a/packages/backend/bin/database/drop.ts b/packages/backend/bin/database/drop.ts index 3c860daf..834d2949 100644 --- a/packages/backend/bin/database/drop.ts +++ b/packages/backend/bin/database/drop.ts @@ -1,29 +1,3 @@ -import appConfig from '../../src/config/app'; -import logger from '../../src/helpers/logger'; -import client from './client'; - -const dropDatabase = async () => { - if (appConfig.appEnv != 'development' && appConfig.appEnv != 'test') { - const errorMessage = 'Drop database command can be used only with development or test environments!' - - logger.error(errorMessage) - return; - } - - await client.connect(); - await dropDatabaseAndUser(); - - await client.end(); -} - -const dropDatabaseAndUser = async() => { - await client.query(`DROP DATABASE IF EXISTS ${appConfig.postgresDatabase}`); - logger.info(`Database: ${appConfig.postgresDatabase} removed!`); - - await client.query(`DROP USER IF EXISTS ${appConfig.postgresUsername}`); - logger.info(`Database User: ${appConfig.postgresUsername} removed!`); -} - - +import { dropDatabase } from './utils'; dropDatabase(); diff --git a/packages/backend/bin/database/seed-user.ts b/packages/backend/bin/database/seed-user.ts index c2a06e1c..0ce56315 100644 --- a/packages/backend/bin/database/seed-user.ts +++ b/packages/backend/bin/database/seed-user.ts @@ -1,15 +1,3 @@ -import User from '../../src/models/user'; -import '../../src/config/orm'; -import logger from '../../src/helpers/logger'; - -const userParams = { - email: 'user@automatisch.io', - password: 'sample', -}; - -async function createUser() { - const user = await User.query().insertAndFetch(userParams); - logger.info(`User has been saved: ${user.email}`); -} +import { createUser } from './utils'; createUser(); diff --git a/packages/backend/bin/database/utils.ts b/packages/backend/bin/database/utils.ts new file mode 100644 index 00000000..aa0b8fa5 --- /dev/null +++ b/packages/backend/bin/database/utils.ts @@ -0,0 +1,100 @@ +import appConfig from '../../src/config/app'; +import logger from '../../src/helpers/logger'; +import client from './client'; +import User from '../../src/models/user'; +import '../../src/config/orm'; + +export async function createUser(email = 'user@automatisch.io', password = 'sample') { + const UNIQUE_VIOLATION_CODE = '23505'; + const userParams = { + email, + password, + }; + + try { + const user = await User.query().insertAndFetch(userParams); + logger.info(`User has been saved: ${user.email}`); + } catch (err) { + if ((err as any).nativeError.code !== UNIQUE_VIOLATION_CODE) { + throw err; + } + + logger.info(`User already exists: ${email}`); + } +} + +export const createDatabaseAndUser = async (database = appConfig.postgresDatabase, user = appConfig.postgresUsername) => { + await client.connect(); + await createDatabase(database); + await createDatabaseUser(user); + await grantPrivileges(database, user); + + await client.end(); +} + +export const createDatabase = async (database = appConfig.postgresDatabase) => { + const DUPLICATE_DB_CODE = '42P04'; + + try { + await client.query(`CREATE DATABASE ${database}`); + logger.info(`Database: ${database} created!`); + } catch (err) { + if ((err as any).code !== DUPLICATE_DB_CODE) { + throw err; + } + + logger.info(`Database: ${database} already exists!`); + } +} + +export const createDatabaseUser = async (user = appConfig.postgresUsername) => { + const DUPLICATE_OBJECT_CODE = '42710'; + + try { + const result = await client.query(`CREATE USER ${user}`); + logger.info(`Database User: ${user} created!`); + + return result; + } catch (err) { + if ((err as any).code !== DUPLICATE_OBJECT_CODE) { + throw err; + } + + logger.info(`Database User: ${user} already exists!`); + } +} + +export const grantPrivileges = async ( + database = appConfig.postgresDatabase, user = appConfig.postgresUsername +) => { + await client.query( + `GRANT ALL PRIVILEGES ON DATABASE ${database} TO ${user};` + ); + + logger.info( + `${user} has granted all privileges on ${database}!` + ); +} + +export const dropDatabase = async () => { + if (appConfig.appEnv != 'development' && appConfig.appEnv != 'test') { + const errorMessage = 'Drop database command can be used only with development or test environments!' + + logger.error(errorMessage) + return; + } + + await client.connect(); + await dropDatabaseAndUser(); + + await client.end(); +} + +export const dropDatabaseAndUser = async(database = appConfig.postgresDatabase, user = appConfig.postgresUsername) => { + await client.query(`DROP DATABASE IF EXISTS ${database}`); + logger.info(`Database: ${database} removed!`); + + await client.query(`DROP USER IF EXISTS ${user}`); + logger.info(`Database User: ${user} removed!`); +} + diff --git a/packages/backend/database.d.ts b/packages/backend/database.d.ts new file mode 100644 index 00000000..5bad32a9 --- /dev/null +++ b/packages/backend/database.d.ts @@ -0,0 +1,2 @@ +export * as utils from './dist/bin/database/utils'; +export * as database from './dist/src/config/database'; diff --git a/packages/backend/database.js b/packages/backend/database.js new file mode 100644 index 00000000..37b36557 --- /dev/null +++ b/packages/backend/database.js @@ -0,0 +1,3 @@ +/* eslint-disable */ +module.exports.utils = require('./dist/bin/database/utils'); +module.exports.database = require('./dist/src/config/database'); diff --git a/packages/backend/logger.d.ts b/packages/backend/logger.d.ts new file mode 100644 index 00000000..1d049ef5 --- /dev/null +++ b/packages/backend/logger.d.ts @@ -0,0 +1 @@ +export * from './dist/src/helpers/logger'; diff --git a/packages/backend/logger.js b/packages/backend/logger.js new file mode 100644 index 00000000..a26b29a1 --- /dev/null +++ b/packages/backend/logger.js @@ -0,0 +1,2 @@ +/* eslint-disable */ +module.exports = require('./dist/src/helpers/logger'); diff --git a/packages/backend/package.json b/packages/backend/package.json index d7f1ec49..6065680b 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -3,9 +3,10 @@ "version": "0.1.0", "description": "> TODO: description", "scripts": { - "dev": "nodemon --watch 'src/**/*.ts' --exec 'ts-node' src/server.ts", + "dev": "nodemon --watch 'src/**/*.ts' --watch 'bin/**/*.ts' --exec 'ts-node' src/server.ts", "worker": "nodemon --watch 'src/**/*.ts' --exec 'ts-node' src/worker.ts", "build": "tsc && yarn copy-statics", + "build:watch": "nodemon --watch 'src/**/*.ts' --watch 'bin/**/*.ts' --exec yarn build --ext ts", "start": "node dist/src/server.js", "test": "ava", "lint": "eslint . --ignore-path ../../.eslintignore", @@ -53,7 +54,7 @@ "twilio": "3.70.0", "twitch-js": "2.0.0-beta.42", "twitter-api-v2": "1.6.0", - "winston": "^3.3.3" + "winston": "^3.7.1" }, "contributors": [ { @@ -64,11 +65,19 @@ "homepage": "https://github.com/automatisch/automatisch#readme", "main": "dist/src/app", "directories": { + "bin": "bin", "src": "src", "test": "__tests__" }, "files": [ - "src" + "bin", + "src", + "server.js", + "server.d.ts", + "logger.js", + "logger.d.ts", + "database.js", + "database.d.ts" ], "repository": { "type": "git", diff --git a/packages/backend/server.d.ts b/packages/backend/server.d.ts new file mode 100644 index 00000000..b940dac2 --- /dev/null +++ b/packages/backend/server.d.ts @@ -0,0 +1 @@ +export * from './dist/src/server'; diff --git a/packages/backend/server.js b/packages/backend/server.js new file mode 100644 index 00000000..821f6b37 --- /dev/null +++ b/packages/backend/server.js @@ -0,0 +1,2 @@ +/* eslint-disable */ +module.exports = require('./dist/src/server.js'); diff --git a/packages/backend/src/config/database.ts b/packages/backend/src/config/database.ts index cf55a2fb..559d5e0c 100644 --- a/packages/backend/src/config/database.ts +++ b/packages/backend/src/config/database.ts @@ -4,16 +4,14 @@ import type { Knex } from 'knex'; import knexConfig from '../../knexfile'; import logger from '../helpers/logger'; -const knexInstance: Knex = knex(knexConfig); +export const client: Knex = knex(knexConfig); const CONNECTION_REFUSED = 'ECONNREFUSED'; -knexInstance.raw('SELECT 1') +client.raw('SELECT 1') .catch((err) => { if (err.code === CONNECTION_REFUSED) { logger.error('Make sure you have installed PostgreSQL and it is running.', err); process.exit(); } }); - -export default knexInstance; diff --git a/packages/backend/src/config/orm.ts b/packages/backend/src/config/orm.ts index a154fad3..77126e91 100644 --- a/packages/backend/src/config/orm.ts +++ b/packages/backend/src/config/orm.ts @@ -1,4 +1,4 @@ import { Model } from 'objection'; -import database from './database'; +import { client } from './database'; -Model.knex(database) +Model.knex(client); diff --git a/packages/backend/src/helpers/logger.ts b/packages/backend/src/helpers/logger.ts index c206bff5..4c27ea64 100644 --- a/packages/backend/src/helpers/logger.ts +++ b/packages/backend/src/helpers/logger.ts @@ -1,5 +1,5 @@ -import winston from 'winston' -import appConfig from '../config/app' +import * as winston from 'winston'; +import appConfig from '../config/app'; const levels = { error: 0, @@ -7,11 +7,11 @@ const levels = { info: 2, http: 3, debug: 4, -} +}; const level = () => { return appConfig.appEnv === 'development' ? 'debug' : 'info' -} +}; const colors = { error: 'red', @@ -19,9 +19,9 @@ const colors = { info: 'green', http: 'magenta', debug: 'white', -} +}; -winston.addColors(colors) +winston.addColors(colors); const format = winston.format.combine( winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss:ms' }), @@ -29,7 +29,7 @@ const format = winston.format.combine( winston.format.printf( (info) => `${info.timestamp} [${info.level}]: ${info.message}`, ), -) +); const transports = [ new winston.transports.Console(), @@ -38,13 +38,13 @@ const transports = [ level: 'error', }), new winston.transports.File({ filename: 'logs/server.log' }), -] +]; -const logger = winston.createLogger({ +export const logger = winston.createLogger({ level: level(), levels, format, transports, -}) +}); -export default logger +export default logger; diff --git a/packages/backend/src/models/connection.ts b/packages/backend/src/models/connection.ts index 997b7ffd..5f8cb669 100644 --- a/packages/backend/src/models/connection.ts +++ b/packages/backend/src/models/connection.ts @@ -13,7 +13,7 @@ class Connection extends Base { formattedData?: IJSONObject; userId!: string; verified = false; - count: number; + count?: number; static tableName = 'connections'; diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index 8485fa63..24c7b911 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -26,10 +26,5 @@ "include": [ "src/**/*", "bin/**/*" - ], - "ts-node": { - "compilerOptions": { - "module": "commonjs" - } - } + ] } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index a66bec9c..2edba74b 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'fs'; import { Command, Flags } from '@oclif/core'; import * as dotenv from 'dotenv'; @@ -16,7 +17,13 @@ export default class Start extends Command { const { flags } = await this.parse(Start); if (flags['env-file']) { - dotenv.config({ path: flags['env-file'] }); + const envFile = readFileSync(flags['env-file'], 'utf8'); + const envConfig = dotenv.parse(envFile); + + for (const key in envConfig) { + const value = envConfig[key]; + process.env[key] = value; + } } if (flags.env) { @@ -26,31 +33,57 @@ export default class Start extends Command { } } + // must serve until more customization is introduced delete process.env.SERVE_WEB_APP_SEPARATELY; } + async createDatabaseAndUser(): Promise { + const { utils } = await import('@automatisch/backend/database'); + + await utils.createDatabaseAndUser( + process.env.POSTGRES_DATABASE, + process.env.POSTGRES_USERNAME, + ); + } + async runMigrationsIfNeeded(): Promise { - const database = (await import('@automatisch/backend/dist/src/config/database')).default; - const migrator = database.migrate; + const { logger } = await import('@automatisch/backend/logger'); + const { database } = await import('@automatisch/backend/database'); + const migrator = database.client.migrate; const [, pendingMigrations] = await migrator.list(); const pendingMigrationsCount = pendingMigrations.length; const needsToMigrate = pendingMigrationsCount > 0; if (needsToMigrate) { + logger.info(`Processing ${pendingMigrationsCount} migrations.`); + await migrator.latest(); + logger.info(`Completed ${pendingMigrationsCount} migrations.`); + } else { + logger.info('No migrations needed.'); } } + async seedUser(): Promise { + const { utils } = await import('@automatisch/backend/database'); + + await utils.createUser(); + } + async runApp(): Promise { - await import('@automatisch/backend/dist/src/server'); + await import('@automatisch/backend/server'); } async run(): Promise { await this.prepareEnvVars(); + await this.createDatabaseAndUser(); + await this.runMigrationsIfNeeded(); + await this.seedUser(); + await this.runApp(); } } diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index c5adb764..61195f40 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -11,7 +11,6 @@ "skipLibCheck": true, "strict": true, "target": "es2021", - "traceResolution": false, "typeRoots": ["node_modules/@types", "./src/types"] }, "include": ["src/**/*"] diff --git a/yarn.lock b/yarn.lock index 7e3051af..c3b2dc0f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1473,6 +1473,11 @@ dependencies: "@bull-board/api" "3.10.1" +"@colors/colors@1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" + integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== + "@concordance/react@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@concordance/react/-/react-2.0.0.tgz#aef913f27474c53731f4fd79cc2f54897de90fde" @@ -12399,6 +12404,17 @@ logform@^2.3.2: safe-stable-stringify "^1.1.0" triple-beam "^1.3.0" +logform@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/logform/-/logform-2.4.0.tgz#131651715a17d50f09c2a2c1a524ff1a4164bcfe" + integrity sha512-CPSJw4ftjf517EhXZGGvTHHkYobo7ZCc0kvwUoOYcjfR2UVrI66RHj8MCrfAdEitdmFqbu2BYdYs8FHHZSb6iw== + dependencies: + "@colors/colors" "1.5.0" + fecha "^4.2.0" + ms "^2.1.1" + safe-stable-stringify "^2.3.1" + triple-beam "^1.3.0" + loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -16411,7 +16427,7 @@ safe-stable-stringify@^1.1.0: resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-1.1.1.tgz#c8a220ab525cd94e60ebf47ddc404d610dc5d84a" integrity sha512-ERq4hUjKDbJfE4+XtZLFPCDi8Vb1JqaxAPTxWFLBx8XcAlf9Bda/ZJdVezs/NAfsMQScyIlUMx+Yeu7P7rx5jw== -safe-stable-stringify@^2.1.0: +safe-stable-stringify@^2.1.0, safe-stable-stringify@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73" integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg== @@ -17943,7 +17959,7 @@ trim@0.0.1: resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" integrity sha1-WFhUf2spB1fulczMZm+1AITEYN0= -triple-beam@^1.2.0, triple-beam@^1.3.0: +triple-beam@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== @@ -19021,29 +19037,30 @@ wildcard@^2.0.0: resolved "https://registry.yarnpkg.com/wildcard/-/wildcard-2.0.0.tgz#a77d20e5200c6faaac979e4b3aadc7b3dd7f8fec" integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== -winston-transport@^4.4.2: - version "4.4.2" - resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.4.2.tgz#554efe3fce229d046df006e0e3c411d240652e51" - integrity sha512-9jmhltAr5ygt5usgUTQbEiw/7RYXpyUbEAFRCSicIacpUzPkrnQsQZSPGEI12aLK9Jth4zNcYJx3Cvznwrl8pw== +winston-transport@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.5.0.tgz#6e7b0dd04d393171ed5e4e4905db265f7ab384fa" + integrity sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q== dependencies: logform "^2.3.2" - readable-stream "^3.4.0" - triple-beam "^1.2.0" + readable-stream "^3.6.0" + triple-beam "^1.3.0" -winston@^3.3.3: - version "3.4.0" - resolved "https://registry.yarnpkg.com/winston/-/winston-3.4.0.tgz#7080f24b02a0684f8a37f9d5c6afb1ac23e95b84" - integrity sha512-FqilVj+5HKwCfIHQzMxrrd5tBIH10JTS3koFGbLVWBODjiIYq7zir08rFyBT4rrTYG/eaTqDcfSIbcjSM78YSw== +winston@^3.7.1: + version "3.7.2" + resolved "https://registry.yarnpkg.com/winston/-/winston-3.7.2.tgz#95b4eeddbec902b3db1424932ac634f887c400b1" + integrity sha512-QziIqtojHBoyzUOdQvQiar1DH0Xp9nF1A1y7NVy2DGEsz82SBDtOalS0ulTRGVT14xPX3WRWkCsdcJKqNflKng== dependencies: "@dabh/diagnostics" "^2.0.2" async "^3.2.3" is-stream "^2.0.0" - logform "^2.3.2" + logform "^2.4.0" one-time "^1.0.0" readable-stream "^3.4.0" + safe-stable-stringify "^2.3.1" stack-trace "0.0.x" triple-beam "^1.3.0" - winston-transport "^4.4.2" + winston-transport "^4.5.0" word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3"