From 48f7f9a68c96ef07c2523923f50f5bed95f59e78 Mon Sep 17 00:00:00 2001 From: Eduardo Szeckir Date: Sun, 11 Jun 2023 21:41:20 -0700 Subject: [PATCH 1/6] adding initial scripts and gh action --- .github/workflows/update-algolia.yaml | 68 +++++++++++++++++++++ functions/scripts/generate-algolia-index.js | 18 ++++++ functions/scripts/generate-courses.js | 60 ++++++++++++++++++ functions/scripts/upload-algolia-index.js | 20 ++++++ 4 files changed, 166 insertions(+) create mode 100644 .github/workflows/update-algolia.yaml create mode 100644 functions/scripts/generate-algolia-index.js create mode 100644 functions/scripts/generate-courses.js create mode 100644 functions/scripts/upload-algolia-index.js diff --git a/.github/workflows/update-algolia.yaml b/.github/workflows/update-algolia.yaml new file mode 100644 index 00000000..d4280fed --- /dev/null +++ b/.github/workflows/update-algolia.yaml @@ -0,0 +1,68 @@ +name: Script - upserts course information to Algolia +on: + worflow_dispatch: + inputs: + environment: + description: Application environment to execute in. + required: true + term: + description: Academic term to execute descript with. ie 202009, 202001 + required: true +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./functions + steps: + - uses: actions/checkout@v2 + - name: Setup Node.js + uses: actions/setup-node@v1 + with: + node-version: 14 + - name: Cache npm dependencies + uses: actions/cache@v2 + with: + key: npm-${{ hashFiles('package-lock.json') }} + path: ~/.npm + restore-keys: | + npm- + - name: Install dependencies + run: npm ci --ignore-scripts --no-audit --no-progress + - name: Lint + run: npm run lint + update-algolia: + name: Update Algolia index + needs: [lint] + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment }} + defaults: + run: + working-directory: ./functions + steps: + - uses: actions/checkout@v2 + - name: Setup Node.js + uses: actions/setup-node@v1 + with: + node-version: 14 + - name: Cache npm dependencies + uses: actions/cache@v2 + with: + key: npm-${{ hashFiles('package-lock.json') }} + path: ~/.npm + restore-keys: | + npm- + - name: Install dependencies + run: npm ci --ignore-scripts --no-audit --no-progress + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }} + export_default_credentials: true + - name: Generate latest course list + run: node ./scripts/generate-courses.js ${{ github.event.inputs.term }} + - name: Generate index for Algolia + run: node ./scripts/generate-algolia-index.js + - name: Upload index to Algolia + run: node ./scripts/upload-algolia-index.js diff --git a/functions/scripts/generate-algolia-index.js b/functions/scripts/generate-algolia-index.js new file mode 100644 index 00000000..7d28e27c --- /dev/null +++ b/functions/scripts/generate-algolia-index.js @@ -0,0 +1,18 @@ +const { writeFile, readFile } = require('fs').promises; + +const main = async () => { + const data = JSON.parse(await readFile('courses.json', 'utf-8')); + + const courses = data.map((course) => ({ + subjectCode: `${course.subject}${course.code}`, + subject: course.subject, + code: course.code, + pid: course.pid, + description: course.description, + title: course.title, + })); + + await writeFile('index.json', JSON.stringify(courses)); +}; + +main(); diff --git a/functions/scripts/generate-courses.js b/functions/scripts/generate-courses.js new file mode 100644 index 00000000..7a8daa85 --- /dev/null +++ b/functions/scripts/generate-courses.js @@ -0,0 +1,60 @@ +const { writeFile } = require('fs').promises; +const fetch = require('node-fetch'); + +if (process.argv.length != 3) throw Error('Term argument not found.'); + +const term = process.argv[2]; + +if (!/20\d{2}0[1,5,9]/.test(term.trim())) + throw Error('Invalid term argument format'); + +const BASE = 'https://courseup.vikelabs.ca/api'; + +const getCourses = async (term) => { + const res = await fetch(`${BASE}/courses/${term}`); + const data = await res.json(); + return data; +}; + +const getCourse = async (term, course) => { + const res = await fetch( + `${BASE}/courses/${term}/${course.subject}/${course.code}` + ); + const data = await res.json(); + return data; +}; + +const getCourseSections = async (term, course) => { + const res = await fetch( + `${BASE}/sections/${term}?subject=${course.subject}&code=${course.code}` + ); + const data = await res.json(); + return data; +}; + +const main = async () => { + const courses = await getCourses(term); + + const values = courses.values(); + + const vals = []; + + const workers = Array(50) + .fill(values) + .map(async (interator, i) => { + for (const course of interator) { + const [courseDetails, courseSections] = await Promise.all([ + getCourse(term, course), + getCourseSections(term, course), + ]); + console.log(course.subject, course.code); + vals.push({ ...courseDetails, sections: courseSections }); + } + }); + + await Promise.allSettled(workers); + + await writeFile('courses.json', JSON.stringify(vals)); +}; + +main(); diff --git a/functions/scripts/upload-algolia-index.js b/functions/scripts/upload-algolia-index.js new file mode 100644 index 00000000..c6e97136 --- /dev/null +++ b/functions/scripts/upload-algolia-index.js @@ -0,0 +1,20 @@ +const algoliasearch = require('algoliasearch'); +const { readFile } = require('fs').promises; + +const client = algoliasearch('ALGOLIA_APPLICATION_ID', 'ALGOLIA_ADMIN_API_KEY'); +const index = client.initIndex('ALGOLIA_INDEX_NAME'); + +const main = async () => { + const data = JSON.parse(await readFile('index.json', 'utf-8')); + + index + .saveObjects(data) + .then(({ objectIDs }) => { + console.log(objectIDs); + }) + .catch((err) => { + console.log(err); + }); +}; + +main(); From af08200dd3ef0d0e4486b1a7243b0d76a9a6ae2f Mon Sep 17 00:00:00 2001 From: Eduardo Szeckir Date: Sun, 11 Jun 2023 21:44:12 -0700 Subject: [PATCH 2/6] fix typos --- .github/workflows/update-algolia.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-algolia.yaml b/.github/workflows/update-algolia.yaml index d4280fed..e9563801 100644 --- a/.github/workflows/update-algolia.yaml +++ b/.github/workflows/update-algolia.yaml @@ -1,12 +1,12 @@ name: Script - upserts course information to Algolia on: - worflow_dispatch: + workflow_dispatch: inputs: environment: description: Application environment to execute in. required: true term: - description: Academic term to execute descript with. ie 202009, 202001 + description: Academic term to execute the script with. ie 202009, 202001 required: true jobs: lint: From 14cd92e4030e84ce2ce4e5f791c4f3d74e43d6a8 Mon Sep 17 00:00:00 2001 From: Eduardo Szeckir Date: Fri, 7 Jul 2023 18:43:50 -0700 Subject: [PATCH 3/6] update to typescript and fix ts errors --- functions/package-lock.json | 96 ++++++++++----------- functions/package.json | 1 + functions/scripts/generate-algolia-index.js | 18 ---- functions/scripts/generate-courses.js | 60 ------------- functions/scripts/update-algolia-index.ts | 76 ++++++++++++++++ functions/scripts/upload-algolia-index.js | 20 ----- functions/tsconfig.dev.json | 2 +- functions/tsconfig.json | 2 +- 8 files changed, 125 insertions(+), 150 deletions(-) delete mode 100644 functions/scripts/generate-algolia-index.js delete mode 100644 functions/scripts/generate-courses.js create mode 100644 functions/scripts/update-algolia-index.ts delete mode 100644 functions/scripts/upload-algolia-index.js diff --git a/functions/package-lock.json b/functions/package-lock.json index 7faced5f..2eb0e5e9 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -27,6 +27,7 @@ "@types/ioredis": "^4.27.1", "@types/jest": "^26.0.24", "@types/node": "^14.18.3", + "@types/node-fetch": "^2.6.4", "@types/object-hash": "^2.1.1", "@types/supertest": "^2.0.11", "@types/swagger-ui-express": "^4.1.3", @@ -2107,6 +2108,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.3.tgz", "integrity": "sha512-GtTH2crF4MtOIrrAa+jgTV9JX/PfoUCYr6MiZw7O/dkZu5b6gm5dc1nAL0jwGo4ortSBBtGyeVaxdC8X6V+pLg==" }, + "node_modules/@types/node-fetch": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz", + "integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz", @@ -5056,6 +5067,20 @@ "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", "dev": true }, + "node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/formidable": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.2.tgz", @@ -8030,20 +8055,6 @@ "node": ">=0.4.0" } }, - "node_modules/jsdom/node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -10482,20 +10493,6 @@ "node": ">= 7.0.0" } }, - "node_modules/superagent/node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/superagent/node_modules/qs": { "version": "6.10.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", @@ -13420,6 +13417,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.3.tgz", "integrity": "sha512-GtTH2crF4MtOIrrAa+jgTV9JX/PfoUCYr6MiZw7O/dkZu5b6gm5dc1nAL0jwGo4ortSBBtGyeVaxdC8X6V+pLg==" }, + "@types/node-fetch": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz", + "integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "@types/normalize-package-data": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz", @@ -15683,6 +15690,17 @@ "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", "dev": true }, + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "formidable": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.2.tgz", @@ -17955,17 +17973,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz", "integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==", "dev": true - }, - "form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } } } }, @@ -19870,17 +19877,6 @@ "semver": "^7.3.2" }, "dependencies": { - "form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } - }, "qs": { "version": "6.10.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", diff --git a/functions/package.json b/functions/package.json index e4b2788a..1f27f9d4 100644 --- a/functions/package.json +++ b/functions/package.json @@ -43,6 +43,7 @@ "@types/ioredis": "^4.27.1", "@types/jest": "^26.0.24", "@types/node": "^14.18.3", + "@types/node-fetch": "^2.6.4", "@types/object-hash": "^2.1.1", "@types/supertest": "^2.0.11", "@types/swagger-ui-express": "^4.1.3", diff --git a/functions/scripts/generate-algolia-index.js b/functions/scripts/generate-algolia-index.js deleted file mode 100644 index 7d28e27c..00000000 --- a/functions/scripts/generate-algolia-index.js +++ /dev/null @@ -1,18 +0,0 @@ -const { writeFile, readFile } = require('fs').promises; - -const main = async () => { - const data = JSON.parse(await readFile('courses.json', 'utf-8')); - - const courses = data.map((course) => ({ - subjectCode: `${course.subject}${course.code}`, - subject: course.subject, - code: course.code, - pid: course.pid, - description: course.description, - title: course.title, - })); - - await writeFile('index.json', JSON.stringify(courses)); -}; - -main(); diff --git a/functions/scripts/generate-courses.js b/functions/scripts/generate-courses.js deleted file mode 100644 index 7a8daa85..00000000 --- a/functions/scripts/generate-courses.js +++ /dev/null @@ -1,60 +0,0 @@ -const { writeFile } = require('fs').promises; -const fetch = require('node-fetch'); - -if (process.argv.length != 3) throw Error('Term argument not found.'); - -const term = process.argv[2]; - -if (!/20\d{2}0[1,5,9]/.test(term.trim())) - throw Error('Invalid term argument format'); - -const BASE = 'https://courseup.vikelabs.ca/api'; - -const getCourses = async (term) => { - const res = await fetch(`${BASE}/courses/${term}`); - const data = await res.json(); - return data; -}; - -const getCourse = async (term, course) => { - const res = await fetch( - `${BASE}/courses/${term}/${course.subject}/${course.code}` - ); - const data = await res.json(); - return data; -}; - -const getCourseSections = async (term, course) => { - const res = await fetch( - `${BASE}/sections/${term}?subject=${course.subject}&code=${course.code}` - ); - const data = await res.json(); - return data; -}; - -const main = async () => { - const courses = await getCourses(term); - - const values = courses.values(); - - const vals = []; - - const workers = Array(50) - .fill(values) - .map(async (interator, i) => { - for (const course of interator) { - const [courseDetails, courseSections] = await Promise.all([ - getCourse(term, course), - getCourseSections(term, course), - ]); - console.log(course.subject, course.code); - vals.push({ ...courseDetails, sections: courseSections }); - } - }); - - await Promise.allSettled(workers); - - await writeFile('courses.json', JSON.stringify(vals)); -}; - -main(); diff --git a/functions/scripts/update-algolia-index.ts b/functions/scripts/update-algolia-index.ts new file mode 100644 index 00000000..fbfda71a --- /dev/null +++ b/functions/scripts/update-algolia-index.ts @@ -0,0 +1,76 @@ +import fetch from 'node-fetch'; +import algoliasearch from 'algoliasearch'; + +if (process.argv.length != 3) throw Error('Term argument not found.'); + +const term: string = process.argv[2]; + +if (!/20\d{2}0[1,5,9]/.test(term.trim())) + throw Error('Invalid term argument format'); + +const BASE = 'https://courseup.vikelabs.ca/api'; + +const client = algoliasearch('U3W5HDPJ0I', 'c82a986b1b3c68f894179887c29a809c'); +const index = client.initIndex('courseup_index'); + +interface Course { + subject: string; + code: string; + pid: string; + description: string; + title: string; +} + +const getCourses = async (term: string): Promise => { + const res = await fetch(`${BASE}/courses/${term}`); + const data: Course[] = await res.json(); + return data; +}; + +const getCourse = async (term: string, course: Course): Promise => { + const res = await fetch( + `${BASE}/courses/${term}/${course.subject}/${course.code}` + ); + const data: Course = await res.json(); + return data; +}; + +const main = async () => { + const courses: Course[] = await getCourses(term); + + const values = courses.values(); + + const vals: Course[] = []; + + const workers = Array(50) + .fill(values) + .map(async (iterator) => { + for (const course of iterator) { + const courseDetails = await getCourse(term, course); + console.log(course.subject, course.code); + vals.push({ ...courseDetails }); + } + }); + + await Promise.allSettled(workers); + + const coursesForIndex = vals.map((course) => ({ + objectID: `${course.subject}${course.code}`, + subject: course.subject, + code: course.code, + pid: course.pid, + description: course.description, + title: course.title, + })); + + index + .saveObjects(coursesForIndex) + .then(({ objectIDs }) => { + console.log(objectIDs); + }) + .catch((err) => { + console.log(err); + }); +}; + +main(); diff --git a/functions/scripts/upload-algolia-index.js b/functions/scripts/upload-algolia-index.js deleted file mode 100644 index c6e97136..00000000 --- a/functions/scripts/upload-algolia-index.js +++ /dev/null @@ -1,20 +0,0 @@ -const algoliasearch = require('algoliasearch'); -const { readFile } = require('fs').promises; - -const client = algoliasearch('ALGOLIA_APPLICATION_ID', 'ALGOLIA_ADMIN_API_KEY'); -const index = client.initIndex('ALGOLIA_INDEX_NAME'); - -const main = async () => { - const data = JSON.parse(await readFile('index.json', 'utf-8')); - - index - .saveObjects(data) - .then(({ objectIDs }) => { - console.log(objectIDs); - }) - .catch((err) => { - console.log(err); - }); -}; - -main(); diff --git a/functions/tsconfig.dev.json b/functions/tsconfig.dev.json index 3b0159b8..dd8aa341 100644 --- a/functions/tsconfig.dev.json +++ b/functions/tsconfig.dev.json @@ -8,7 +8,7 @@ "outDir": "lib", "sourceMap": true, "strict": true, - "target": "ES2019" + "target": "es2020" }, "include": ["scripts", "src", ".eslintrc.js"] } diff --git a/functions/tsconfig.json b/functions/tsconfig.json index cdc155d1..21608e86 100644 --- a/functions/tsconfig.json +++ b/functions/tsconfig.json @@ -8,7 +8,7 @@ "outDir": "lib", "sourceMap": true, "strict": true, - "target": "ES2019" + "target": "es2020" }, "compileOnSave": true, "include": ["src"], From c92ab90865c40121e2d8d6772bce1b60c48f6e30 Mon Sep 17 00:00:00 2001 From: Eduardo Szeckir Date: Fri, 7 Jul 2023 18:46:04 -0700 Subject: [PATCH 4/6] update gh action --- .github/workflows/update-algolia.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update-algolia.yaml b/.github/workflows/update-algolia.yaml index e9563801..446eab5d 100644 --- a/.github/workflows/update-algolia.yaml +++ b/.github/workflows/update-algolia.yaml @@ -60,9 +60,5 @@ jobs: with: service_account_key: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }} export_default_credentials: true - - name: Generate latest course list - run: node ./scripts/generate-courses.js ${{ github.event.inputs.term }} - - name: Generate index for Algolia - run: node ./scripts/generate-algolia-index.js - - name: Upload index to Algolia - run: node ./scripts/upload-algolia-index.js + - name: Run Update Algolia Script + run: npx ts-node ./scripts/update-algolia-index.ts ${{ github.event.inputs.term }} From 49ccbe107fa180e917850d7cac42d6848f87fd25 Mon Sep 17 00:00:00 2001 From: Eduardo Szeckir Date: Fri, 7 Jul 2023 19:08:07 -0700 Subject: [PATCH 5/6] set general values --- functions/scripts/update-algolia-index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/functions/scripts/update-algolia-index.ts b/functions/scripts/update-algolia-index.ts index fbfda71a..cd2b7e14 100644 --- a/functions/scripts/update-algolia-index.ts +++ b/functions/scripts/update-algolia-index.ts @@ -10,8 +10,8 @@ if (!/20\d{2}0[1,5,9]/.test(term.trim())) const BASE = 'https://courseup.vikelabs.ca/api'; -const client = algoliasearch('U3W5HDPJ0I', 'c82a986b1b3c68f894179887c29a809c'); -const index = client.initIndex('courseup_index'); +const client = algoliasearch('ALGOLIA_APP_ID', 'ALGOLIA_ADMIN_KEY'); +const index = client.initIndex('ALGOLIA_INDEX_NAME'); interface Course { subject: string; @@ -38,6 +38,8 @@ const getCourse = async (term: string, course: Course): Promise => { const main = async () => { const courses: Course[] = await getCourses(term); + console.log(`Found ${courses.length} courses for ${term}`); + const values = courses.values(); const vals: Course[] = []; From 5e355fb966bb3ed428186c5f078c5de0bcb3184f Mon Sep 17 00:00:00 2001 From: Eduardo Szeckir Date: Mon, 17 Jul 2023 19:26:29 -0700 Subject: [PATCH 6/6] install package, env vars, and remove gh action step --- .github/workflows/update-algolia.yaml | 28 --- functions/package-lock.json | 273 ++++++++++++++++++++++ functions/package.json | 1 + functions/scripts/update-algolia-index.ts | 12 +- 4 files changed, 284 insertions(+), 30 deletions(-) diff --git a/.github/workflows/update-algolia.yaml b/.github/workflows/update-algolia.yaml index 446eab5d..ba10d7cd 100644 --- a/.github/workflows/update-algolia.yaml +++ b/.github/workflows/update-algolia.yaml @@ -9,29 +9,6 @@ on: description: Academic term to execute the script with. ie 202009, 202001 required: true jobs: - lint: - name: Lint - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./functions - steps: - - uses: actions/checkout@v2 - - name: Setup Node.js - uses: actions/setup-node@v1 - with: - node-version: 14 - - name: Cache npm dependencies - uses: actions/cache@v2 - with: - key: npm-${{ hashFiles('package-lock.json') }} - path: ~/.npm - restore-keys: | - npm- - - name: Install dependencies - run: npm ci --ignore-scripts --no-audit --no-progress - - name: Lint - run: npm run lint update-algolia: name: Update Algolia index needs: [lint] @@ -55,10 +32,5 @@ jobs: npm- - name: Install dependencies run: npm ci --ignore-scripts --no-audit --no-progress - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }} - export_default_credentials: true - name: Run Update Algolia Script run: npx ts-node ./scripts/update-algolia-index.ts ${{ github.event.inputs.term }} diff --git a/functions/package-lock.json b/functions/package-lock.json index 2eb0e5e9..b0d58cef 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -7,6 +7,7 @@ "name": "functions", "dependencies": { "@vikelabs/uvic-course-scraper": "^1.0.0-alpha.8", + "algoliasearch": "^4.18.0", "async": "^3.2.1", "express": "^4.17.2", "firebase-admin": "^10.0.1", @@ -58,6 +59,121 @@ "node": "14" } }, + "node_modules/@algolia/cache-browser-local-storage": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.18.0.tgz", + "integrity": "sha512-rUAs49NLlO8LVLgGzM4cLkw8NJLKguQLgvFmBEe3DyzlinoqxzQMHfKZs6TSq4LZfw/z8qHvRo8NcTAAUJQLcw==", + "dependencies": { + "@algolia/cache-common": "4.18.0" + } + }, + "node_modules/@algolia/cache-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.18.0.tgz", + "integrity": "sha512-BmxsicMR4doGbeEXQu8yqiGmiyvpNvejYJtQ7rvzttEAMxOPoWEHrWyzBQw4x7LrBY9pMrgv4ZlUaF8PGzewHg==" + }, + "node_modules/@algolia/cache-in-memory": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.18.0.tgz", + "integrity": "sha512-evD4dA1nd5HbFdufBxLqlJoob7E2ozlqJZuV3YlirNx5Na4q1LckIuzjNYZs2ddLzuTc/Xd5O3Ibf7OwPskHxw==", + "dependencies": { + "@algolia/cache-common": "4.18.0" + } + }, + "node_modules/@algolia/client-account": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.18.0.tgz", + "integrity": "sha512-XsDnlROr3+Z1yjxBJjUMfMazi1V155kVdte6496atvBgOEtwCzTs3A+qdhfsAnGUvaYfBrBkL0ThnhMIBCGcew==", + "dependencies": { + "@algolia/client-common": "4.18.0", + "@algolia/client-search": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.18.0.tgz", + "integrity": "sha512-chEUSN4ReqU7uRQ1C8kDm0EiPE+eJeAXiWcBwLhEynfNuTfawN9P93rSZktj7gmExz0C8XmkbBU19IQ05wCNrQ==", + "dependencies": { + "@algolia/client-common": "4.18.0", + "@algolia/client-search": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "node_modules/@algolia/client-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.18.0.tgz", + "integrity": "sha512-7N+soJFP4wn8tjTr3MSUT/U+4xVXbz4jmeRfWfVAzdAbxLAQbHa0o/POSdTvQ8/02DjCLelloZ1bb4ZFVKg7Wg==", + "dependencies": { + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.18.0.tgz", + "integrity": "sha512-+PeCjODbxtamHcPl+couXMeHEefpUpr7IHftj4Y4Nia1hj8gGq4VlIcqhToAw8YjLeCTfOR7r7xtj3pJcYdP8A==", + "dependencies": { + "@algolia/client-common": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "node_modules/@algolia/client-search": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.18.0.tgz", + "integrity": "sha512-F9xzQXTjm6UuZtnsLIew6KSraXQ0AzS/Ee+OD+mQbtcA/K1sg89tqb8TkwjtiYZ0oij13u3EapB3gPZwm+1Y6g==", + "dependencies": { + "@algolia/client-common": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "node_modules/@algolia/logger-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.18.0.tgz", + "integrity": "sha512-46etYgSlkoKepkMSyaoriSn2JDgcrpc/nkOgou/lm0y17GuMl9oYZxwKKTSviLKI5Irk9nSKGwnBTQYwXOYdRg==" + }, + "node_modules/@algolia/logger-console": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.18.0.tgz", + "integrity": "sha512-3P3VUYMl9CyJbi/UU1uUNlf6Z8N2ltW3Oqhq/nR7vH0CjWv32YROq3iGWGxB2xt3aXobdUPXs6P0tHSKRmNA6g==", + "dependencies": { + "@algolia/logger-common": "4.18.0" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.18.0.tgz", + "integrity": "sha512-/AcWHOBub2U4TE/bPi4Gz1XfuLK6/7dj4HJG+Z2SfQoS1RjNLshZclU3OoKIkFp8D2NC7+BNsPvr9cPLyW8nyQ==", + "dependencies": { + "@algolia/requester-common": "4.18.0" + } + }, + "node_modules/@algolia/requester-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.18.0.tgz", + "integrity": "sha512-xlT8R1qYNRBCi1IYLsx7uhftzdfsLPDGudeQs+xvYB4sQ3ya7+ppolB/8m/a4F2gCkEO6oxpp5AGemM7kD27jA==" + }, + "node_modules/@algolia/requester-node-http": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.18.0.tgz", + "integrity": "sha512-TGfwj9aeTVgOUhn5XrqBhwUhUUDnGIKlI0kCBMdR58XfXcfdwomka+CPIgThRbfYw04oQr31A6/95ZH2QVJ9UQ==", + "dependencies": { + "@algolia/requester-common": "4.18.0" + } + }, + "node_modules/@algolia/transporter": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.18.0.tgz", + "integrity": "sha512-xbw3YRUGtXQNG1geYFEDDuFLZt4Z8YNKbamHPkzr3rWc6qp4/BqEeXcI2u/P/oMq2yxtXgMxrCxOPA8lyIe5jw==", + "dependencies": { + "@algolia/cache-common": "4.18.0", + "@algolia/logger-common": "4.18.0", + "@algolia/requester-common": "4.18.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.14.5.tgz", @@ -2519,6 +2635,27 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/algoliasearch": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.18.0.tgz", + "integrity": "sha512-pCuVxC1SVcpc08ENH32T4sLKSyzoU7TkRIDBMwSLfIiW+fq4znOmWDkAygHZ6pRcO9I1UJdqlfgnV7TRj+MXrA==", + "dependencies": { + "@algolia/cache-browser-local-storage": "4.18.0", + "@algolia/cache-common": "4.18.0", + "@algolia/cache-in-memory": "4.18.0", + "@algolia/client-account": "4.18.0", + "@algolia/client-analytics": "4.18.0", + "@algolia/client-common": "4.18.0", + "@algolia/client-personalization": "4.18.0", + "@algolia/client-search": "4.18.0", + "@algolia/logger-common": "4.18.0", + "@algolia/logger-console": "4.18.0", + "@algolia/requester-browser-xhr": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/requester-node-http": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, "node_modules/ansi-align": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", @@ -11702,6 +11839,121 @@ } }, "dependencies": { + "@algolia/cache-browser-local-storage": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.18.0.tgz", + "integrity": "sha512-rUAs49NLlO8LVLgGzM4cLkw8NJLKguQLgvFmBEe3DyzlinoqxzQMHfKZs6TSq4LZfw/z8qHvRo8NcTAAUJQLcw==", + "requires": { + "@algolia/cache-common": "4.18.0" + } + }, + "@algolia/cache-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.18.0.tgz", + "integrity": "sha512-BmxsicMR4doGbeEXQu8yqiGmiyvpNvejYJtQ7rvzttEAMxOPoWEHrWyzBQw4x7LrBY9pMrgv4ZlUaF8PGzewHg==" + }, + "@algolia/cache-in-memory": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.18.0.tgz", + "integrity": "sha512-evD4dA1nd5HbFdufBxLqlJoob7E2ozlqJZuV3YlirNx5Na4q1LckIuzjNYZs2ddLzuTc/Xd5O3Ibf7OwPskHxw==", + "requires": { + "@algolia/cache-common": "4.18.0" + } + }, + "@algolia/client-account": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.18.0.tgz", + "integrity": "sha512-XsDnlROr3+Z1yjxBJjUMfMazi1V155kVdte6496atvBgOEtwCzTs3A+qdhfsAnGUvaYfBrBkL0ThnhMIBCGcew==", + "requires": { + "@algolia/client-common": "4.18.0", + "@algolia/client-search": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "@algolia/client-analytics": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.18.0.tgz", + "integrity": "sha512-chEUSN4ReqU7uRQ1C8kDm0EiPE+eJeAXiWcBwLhEynfNuTfawN9P93rSZktj7gmExz0C8XmkbBU19IQ05wCNrQ==", + "requires": { + "@algolia/client-common": "4.18.0", + "@algolia/client-search": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "@algolia/client-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.18.0.tgz", + "integrity": "sha512-7N+soJFP4wn8tjTr3MSUT/U+4xVXbz4jmeRfWfVAzdAbxLAQbHa0o/POSdTvQ8/02DjCLelloZ1bb4ZFVKg7Wg==", + "requires": { + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "@algolia/client-personalization": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.18.0.tgz", + "integrity": "sha512-+PeCjODbxtamHcPl+couXMeHEefpUpr7IHftj4Y4Nia1hj8gGq4VlIcqhToAw8YjLeCTfOR7r7xtj3pJcYdP8A==", + "requires": { + "@algolia/client-common": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "@algolia/client-search": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.18.0.tgz", + "integrity": "sha512-F9xzQXTjm6UuZtnsLIew6KSraXQ0AzS/Ee+OD+mQbtcA/K1sg89tqb8TkwjtiYZ0oij13u3EapB3gPZwm+1Y6g==", + "requires": { + "@algolia/client-common": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, + "@algolia/logger-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.18.0.tgz", + "integrity": "sha512-46etYgSlkoKepkMSyaoriSn2JDgcrpc/nkOgou/lm0y17GuMl9oYZxwKKTSviLKI5Irk9nSKGwnBTQYwXOYdRg==" + }, + "@algolia/logger-console": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.18.0.tgz", + "integrity": "sha512-3P3VUYMl9CyJbi/UU1uUNlf6Z8N2ltW3Oqhq/nR7vH0CjWv32YROq3iGWGxB2xt3aXobdUPXs6P0tHSKRmNA6g==", + "requires": { + "@algolia/logger-common": "4.18.0" + } + }, + "@algolia/requester-browser-xhr": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.18.0.tgz", + "integrity": "sha512-/AcWHOBub2U4TE/bPi4Gz1XfuLK6/7dj4HJG+Z2SfQoS1RjNLshZclU3OoKIkFp8D2NC7+BNsPvr9cPLyW8nyQ==", + "requires": { + "@algolia/requester-common": "4.18.0" + } + }, + "@algolia/requester-common": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.18.0.tgz", + "integrity": "sha512-xlT8R1qYNRBCi1IYLsx7uhftzdfsLPDGudeQs+xvYB4sQ3ya7+ppolB/8m/a4F2gCkEO6oxpp5AGemM7kD27jA==" + }, + "@algolia/requester-node-http": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.18.0.tgz", + "integrity": "sha512-TGfwj9aeTVgOUhn5XrqBhwUhUUDnGIKlI0kCBMdR58XfXcfdwomka+CPIgThRbfYw04oQr31A6/95ZH2QVJ9UQ==", + "requires": { + "@algolia/requester-common": "4.18.0" + } + }, + "@algolia/transporter": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.18.0.tgz", + "integrity": "sha512-xbw3YRUGtXQNG1geYFEDDuFLZt4Z8YNKbamHPkzr3rWc6qp4/BqEeXcI2u/P/oMq2yxtXgMxrCxOPA8lyIe5jw==", + "requires": { + "@algolia/cache-common": "4.18.0", + "@algolia/logger-common": "4.18.0", + "@algolia/requester-common": "4.18.0" + } + }, "@babel/code-frame": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.14.5.tgz", @@ -13722,6 +13974,27 @@ "uri-js": "^4.2.2" } }, + "algoliasearch": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.18.0.tgz", + "integrity": "sha512-pCuVxC1SVcpc08ENH32T4sLKSyzoU7TkRIDBMwSLfIiW+fq4znOmWDkAygHZ6pRcO9I1UJdqlfgnV7TRj+MXrA==", + "requires": { + "@algolia/cache-browser-local-storage": "4.18.0", + "@algolia/cache-common": "4.18.0", + "@algolia/cache-in-memory": "4.18.0", + "@algolia/client-account": "4.18.0", + "@algolia/client-analytics": "4.18.0", + "@algolia/client-common": "4.18.0", + "@algolia/client-personalization": "4.18.0", + "@algolia/client-search": "4.18.0", + "@algolia/logger-common": "4.18.0", + "@algolia/logger-console": "4.18.0", + "@algolia/requester-browser-xhr": "4.18.0", + "@algolia/requester-common": "4.18.0", + "@algolia/requester-node-http": "4.18.0", + "@algolia/transporter": "4.18.0" + } + }, "ansi-align": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", diff --git a/functions/package.json b/functions/package.json index 1f27f9d4..f828f5d8 100644 --- a/functions/package.json +++ b/functions/package.json @@ -23,6 +23,7 @@ "main": "lib/src/index.js", "dependencies": { "@vikelabs/uvic-course-scraper": "^1.0.0-alpha.8", + "algoliasearch": "^4.18.0", "async": "^3.2.1", "express": "^4.17.2", "firebase-admin": "^10.0.1", diff --git a/functions/scripts/update-algolia-index.ts b/functions/scripts/update-algolia-index.ts index cd2b7e14..7527febe 100644 --- a/functions/scripts/update-algolia-index.ts +++ b/functions/scripts/update-algolia-index.ts @@ -10,8 +10,16 @@ if (!/20\d{2}0[1,5,9]/.test(term.trim())) const BASE = 'https://courseup.vikelabs.ca/api'; -const client = algoliasearch('ALGOLIA_APP_ID', 'ALGOLIA_ADMIN_KEY'); -const index = client.initIndex('ALGOLIA_INDEX_NAME'); +const ALGOLIA_APP_ID = process.env.ALGOLIA_APP_ID; +const ALGOLIA_ADMIN_KEY = process.env.ALGOLIA_ADMIN_KEY; +const ALGOLIA_INDEX_NAME = process.env.ALGOLIA_INDEX_NAME; + +if (!ALGOLIA_APP_ID || !ALGOLIA_ADMIN_KEY || !ALGOLIA_INDEX_NAME) { + throw Error('Algolia environment variables not found.'); +} + +const client = algoliasearch(ALGOLIA_APP_ID, ALGOLIA_ADMIN_KEY); +const index = client.initIndex(ALGOLIA_INDEX_NAME); interface Course { subject: string;