From 5c265c04ad513d845a41c7866c3ed231c8d5e68e Mon Sep 17 00:00:00 2001 From: Noah Loomans Date: Sat, 7 Jul 2018 17:34:02 +0200 Subject: server: Use database for users --- .eslintrc | 1 + .vscode/settings.json | 3 + package-lock.json | 97 ++++++++++- package.json | 1 + src/client/.eslintrc | 5 + src/client/react/components/container/HelpBox.js | 2 + src/client/views/index.handlebars | 5 +- src/server/lib/parseSchedule.js | 120 +++++++++++++ src/server/lib/schools/hetmml/axios.js | 35 ---- src/server/lib/schools/hetmml/getScheduleData.js | 206 ----------------------- src/server/lib/schools/hetmml/getURLOfUser.js | 27 --- src/server/lib/schools/hetmml/parseSchedule.js | 120 ------------- src/server/routes/getSchedule.js | 16 +- src/server/routes/index.js | 26 +-- src/shared/lib/axios.js | 35 ++++ src/shared/lib/db.js | 62 +++++++ src/shared/lib/getURLOfUser.js | 35 ++++ src/sync/index.js | 97 +++++++++++ src/sync/scrapeScheduleData.js | 89 ++++++++++ 19 files changed, 556 insertions(+), 426 deletions(-) create mode 100644 src/client/.eslintrc create mode 100644 src/server/lib/parseSchedule.js delete mode 100644 src/server/lib/schools/hetmml/axios.js delete mode 100644 src/server/lib/schools/hetmml/getScheduleData.js delete mode 100644 src/server/lib/schools/hetmml/getURLOfUser.js delete mode 100644 src/server/lib/schools/hetmml/parseSchedule.js create mode 100644 src/shared/lib/axios.js create mode 100644 src/shared/lib/db.js create mode 100644 src/shared/lib/getURLOfUser.js create mode 100644 src/sync/index.js create mode 100644 src/sync/scrapeScheduleData.js diff --git a/.eslintrc b/.eslintrc index 36ded27..08e0b82 100644 --- a/.eslintrc +++ b/.eslintrc @@ -14,6 +14,7 @@ "react/forbid-prop-types": "off", "react/prefer-stateless-function": "off", "import/prefer-default-export": "off", + "no-console": "off", "notice/notice":["error", { "templateFile": ".licence-template" }] diff --git a/.vscode/settings.json b/.vscode/settings.json index abf82c4..95419b3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,10 +13,13 @@ // Code Spell Checker "cSpell.language": "en,nl", "cSpell.words": [ + "PGDATABASE", "decrementing", "devtools", "dompurify", "hetmml", + "meetingpoint", + "meetingpointmco", "metis", "polyfill", "presentational", diff --git a/package-lock.json b/package-lock.json index 471a426..43b84a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2242,6 +2242,11 @@ "integrity": "sha512-c5mRlguI/Pe2dSZmpER62rSCu0ryKmWddzRYsuXc50U2/g8jMOulc31VZMa4mYx31U5xsmSOpDCgH88Vl9cDGQ==", "dev": true }, + "buffer-writer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz", + "integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg=" + }, "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", @@ -8175,6 +8180,11 @@ "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", "dev": true }, + "packet-reader": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz", + "integrity": "sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc=" + }, "pako": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz", @@ -8294,6 +8304,56 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, + "pg": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.4.3.tgz", + "integrity": "sha1-97b5P1NA7MJZavu5ShPj1rYJg0s=", + "requires": { + "buffer-writer": "1.0.1", + "packet-reader": "0.3.1", + "pg-connection-string": "0.1.3", + "pg-pool": "2.0.3", + "pg-types": "1.12.1", + "pgpass": "1.0.2", + "semver": "4.3.2" + }, + "dependencies": { + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + } + } + }, + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + }, + "pg-pool": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.3.tgz", + "integrity": "sha1-wCIDLIlJ8xKk+R+2QJzgQHa+Mlc=" + }, + "pg-types": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.12.1.tgz", + "integrity": "sha1-1kCH45A7WP+q0nnnWVxSIIoUw9I=", + "requires": { + "postgres-array": "1.0.2", + "postgres-bytea": "1.0.0", + "postgres-date": "1.0.3", + "postgres-interval": "1.1.1" + } + }, + "pgpass": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", + "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "requires": { + "split": "1.0.1" + } + }, "pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", @@ -8840,6 +8900,29 @@ "uniqs": "2.0.0" } }, + "postgres-array": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.2.tgz", + "integrity": "sha1-jgsy6wO/d6XAp4UeBEHBaaJWojg=" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz", + "integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g=" + }, + "postgres-interval": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.1.tgz", + "integrity": "sha512-OkuCi9t/3CZmeQreutGgx/OVNv9MKHGIT5jH8KldQ4NLYXkvmT9nDVxEuCENlNwhlGPE374oA/xMqn05G49pHA==", + "requires": { + "xtend": "4.0.1" + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -10171,6 +10254,14 @@ "integrity": "sha512-2+EPwgbnmOIl8HjGBXXMd9NAu02vLjOO1nWw4kmeRDFyHn+M/ETfHxQUK0oXg8ctgVnl9t3rosNVsZ1jG61nDA==", "dev": true }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2.3.8" + } + }, "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", @@ -10523,8 +10614,7 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "timers-browserify": { "version": "2.0.10", @@ -11414,8 +11504,7 @@ "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", - "dev": true + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" }, "y18n": { "version": "3.2.1", diff --git a/package.json b/package.json index 5b751f4..7973428 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "lodash": "^4.15.0", "moment": "^2.20.1", "morgan": "^1.9.0", + "pg": "^7.4.3", "promise-debounce": "^1.0.1", "prop-types": "^15.6.0", "query-string": "^6.1.0", diff --git a/src/client/.eslintrc b/src/client/.eslintrc new file mode 100644 index 0000000..6479217 --- /dev/null +++ b/src/client/.eslintrc @@ -0,0 +1,5 @@ +{ + "rules": { + "no-console": "warn" + } +} diff --git a/src/client/react/components/container/HelpBox.js b/src/client/react/components/container/HelpBox.js index d10c26b..f4fbbe9 100644 --- a/src/client/react/components/container/HelpBox.js +++ b/src/client/react/components/container/HelpBox.js @@ -21,6 +21,8 @@ import { connect } from 'react-redux'; import HelpBox from '../presentational/HelpBox'; +console.log('hi'); + const mapStateToProps = state => ({ isVisible: !state.search || (state.search.results.length === 0 && state.search.text === ''), }); diff --git a/src/client/views/index.handlebars b/src/client/views/index.handlebars index 5105439..93eccf1 100644 --- a/src/client/views/index.handlebars +++ b/src/client/views/index.handlebars @@ -28,10 +28,7 @@
diff --git a/src/server/lib/parseSchedule.js b/src/server/lib/parseSchedule.js new file mode 100644 index 0000000..066d637 --- /dev/null +++ b/src/server/lib/parseSchedule.js @@ -0,0 +1,120 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const { JSDOM } = require('jsdom'); + +function fixFirstLargeScheduleItem(trNodeList) { + return Array.from(trNodeList).some((trNode, timeOfDay) => { + const tdNodeList = trNode.children; + + return Array.from(tdNodeList).some((tdNode, dayOfWeek) => { + const height = tdNode.rowSpan / 2; + if (height === 1) { + return false; + } + + tdNode.rowSpan = 2; // eslint-disable-line no-param-reassign + + for (let i = 1; i < height; i += 1) { + // Are we at the end of the table? + if (dayOfWeek === 4) { + // If so, we cannot use insertBefore, because the is no node to insert + // it before. Use appendChild instead. + trNodeList[timeOfDay + i].appendChild(tdNode.cloneNode(true)); + } else { + trNodeList[timeOfDay + i] + .insertBefore( + tdNode.cloneNode(true), + trNodeList[timeOfDay + i].children[dayOfWeek], + ); + } + } + + return true; + }); + }); +} + +function parseSchedule(axiosResponse) { + const dom = new JSDOM(axiosResponse.data); + const { window } = dom; + const { document } = window; + + const tableNode = document.querySelector('center > table'); + const tbodyNode = tableNode.querySelector('tbody'); + const trNodeList = tbodyNode.children; + + Array.from(trNodeList).forEach((trNode, timeOfDay) => { + const tdNodeList = trNode.children; + + if (timeOfDay === 0 || trNode.children.length === 0) { + tbodyNode.removeChild(trNode); + return; + } + + Array.from(tdNodeList).forEach((tdNode, dayOfWeek) => { + if (dayOfWeek === 0) { + trNode.removeChild(tdNode); + } + }); + }); + + let shouldContinue = true; + while (shouldContinue) { + shouldContinue = fixFirstLargeScheduleItem(trNodeList); + } + + const scheduleItems = []; + + Array.from(trNodeList).forEach((trNode, timeOfDay) => { + const tdNodeList = trNode.children; + Array.from(tdNodeList).forEach((tdNode, dayOfWeek) => { + if (tdNode.textContent.trim() === '') { + return; + } + + const childTableNode = tdNode.querySelector('table'); + const childTrNodeList = childTableNode.querySelectorAll('tr'); + + Array.from(childTrNodeList).forEach((childTrNode) => { + const subject = childTrNode.children[0].textContent.trim(); + const attendees = childTrNode.children[1] + ? childTrNode.children[1].textContent.trim() + : undefined; + const location = childTrNode.children[2] + ? childTrNode.children[2].textContent.trim() + : undefined; + + scheduleItems.push({ + startTime: timeOfDay, + endTime: timeOfDay + 1, + dayOfWeek, + subject, + attendees, + location, + }); + }); + }); + }); + + return scheduleItems; +} + +module.exports = parseSchedule; diff --git a/src/server/lib/schools/hetmml/axios.js b/src/server/lib/schools/hetmml/axios.js deleted file mode 100644 index 2de6047..0000000 --- a/src/server/lib/schools/hetmml/axios.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright (C) 2018 Noah Loomans - * - * This file is part of rooster.hetmml.nl. - * - * rooster.hetmml.nl is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * rooster.hetmml.nl is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with rooster.hetmml.nl. If not, see . - * - */ - -const axios = require('axios'); -const iconv = require('iconv-lite'); - -/** - * Create a new axios instance with sensible defaults for connecting to - * meetingpointmco.nl -*/ -const instance = axios.create({ - baseURL: 'http://www.meetingpointmco.nl/Roosters-AL/doc/', - timeout: 5000, - responseType: 'arraybuffer', - transformResponse: [responseBody => iconv.decode(responseBody, 'iso-8859-1')], -}); - -module.exports = instance; diff --git a/src/server/lib/schools/hetmml/getScheduleData.js b/src/server/lib/schools/hetmml/getScheduleData.js deleted file mode 100644 index ead856f..0000000 --- a/src/server/lib/schools/hetmml/getScheduleData.js +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright (C) 2018 Noah Loomans - * - * This file is part of rooster.hetmml.nl. - * - * rooster.hetmml.nl is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * rooster.hetmml.nl is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with rooster.hetmml.nl. If not, see . - * - */ - -const cheerio = require('cheerio'); -const debounce = require('promise-debounce'); -const _ = require('lodash'); - -const getUrlOfUser = require('./getURLOfUser'); -const axios = require('./axios'); - -let meetingpointData; -let lastUpdate; - -/** - * Scrape all the valid users from a meetingpoint navbar. - * @param {string} html The html of a meetingpoint navbar. - * @returns {*} - * [ - * { type: 't', value: 'akh', index: 0 }, - * ... - * { type: 's', value: '18561', index: 245 }, - * ... - * { type: 'r', value: '008-mk', index: 2 }, - * ... - * { type: 'c', value: '6-5H2', index: 23 }, - * ... - * ] - */ -function scrapeUsers(html) { - const page = cheerio.load(html); - const script = page('script').eq(1).text(); - - const regexs = [/var classes = \[(.+)\];/, /var teachers = \[(.+)\];/, /var rooms = \[(.+)\];/, /var students = \[(.+)\];/]; - const items = regexs.map(regex => script.match(regex)[1].split(',').map(item => item.replace(/"/g, ''))); - - const classes = items[0].map((item, index) => ({ - type: 'c', - value: item, - index, - })); - - const teachers = items[1].map((item, index) => ({ - type: 't', - value: item, - index, - })); - - const rooms = items[2].map((item, index) => ({ - type: 'r', - value: item, - index, - })); - - const students = items[3].map((item, index) => ({ - type: 's', - value: item, - index, - })); - - return _.flatten([classes, teachers, rooms, students]); -} - -/** - * Scrape the known valid weeks from a meetingpoint navbar. - * - * There probably are more valid weeks, but these once are garanteed to be - * valid. - * @param {string} html The html of a meetingpoint navbar. - * @returns {*} [{ id: string, text: string }, ...] - */ -function scrapeWeeks(html) { - const page = cheerio.load(html); - const weekSelector = page('select[name="week"]'); - const weeks = _.map(weekSelector.children(), option => ({ - id: cheerio(option).attr('value'), - text: cheerio(option).text(), - })); - - return weeks; -} - -/** - * scrape the alt text (the text next to the short code) from a - * specific meetingpoint schedule. - * @param {string} html The html of a specific meetingpoint schedule. - * @returns {string} - */ -function scrapeAltText(html) { - const page = cheerio.load(html); - return page('center > font').eq(2).text().trim(); -} - -/** - * Combines two user array, if a dublicate user is present, the first one will - * be used. - * - * This function is currently used to merge a subset of users with alts - * attached to them with a compleat set of users without alts. - * @param {*} usersArrays An array of user arrays. - */ -function combineUsers(usersArrays) { - return _.uniqBy(_.flatten(usersArrays), user => `${user.type}/${user.value}`); -} - -/** - * Requests and adds an alt field to the given users. - * - * For example, it will add the teacher name to a teacher object. - * - * @param {*} users [{ type: string, value: string, index: number }, ...] - * @returns {*} [{ type: string, value: string, alt: string, index: number }, ...] - */ -function getAlts(users) { - const requests = users.map(user => ( - axios.get( - getUrlOfUser('dag', user.type, user.index, 7), - { timeout: 8000 }, - ) - )); - - return Promise.all(requests).then(teacherResponses => ( - teacherResponses.map((teacherResponse, index) => { - const teacherName = scrapeAltText(teacherResponse.data); - - return { - ...users[index], - alt: teacherName, - }; - }) - )); -} - -/** - * Requests all the relevent data from the meetingpoint server - * This is very expensive! Only call when you absolutely need to. - * @returns {Promise} { users, dailyScheduleWeeks, basisScheduleWeeks } - */ -function getScheduleData() { - const navbarRequests = [ - axios.get('/dagroosters/frames/navbar.htm'), - axios.get('/basisroosters/frames/navbar.htm'), - ]; - - return Promise.all(navbarRequests) - .then(([dailyScheduleResponse, basisScheduleResponse]) => { - const users = scrapeUsers(dailyScheduleResponse.data); - const dailyScheduleWeeks = scrapeWeeks(dailyScheduleResponse.data); - const basisScheduleWeeks = scrapeWeeks(basisScheduleResponse.data); - - const teachers = users.filter(user => user.type === 't'); - - return getAlts(teachers) - .then(teachersWithAlts => ({ - users: combineUsers([teachersWithAlts, users]), - dailyScheduleWeeks, - basisScheduleWeeks, - })) - .catch(() => ({ - // Just return the user data without the alts if getAlts fails, since - // the alts are non-essential. - users, - dailyScheduleWeeks, - basisScheduleWeeks, - })); - }); -} - -/** - * Wrapper around getScheduleData that is cheap to call. In most cases it - * returns a cached version. The cache is stored for 30 minutes. - * @returns {Promise} { users, dailyScheduleWeeks, basisScheduleWeeks } -*/ -function getScheduleDataCacheWrapper() { - if (meetingpointData == null || new Date() - lastUpdate > 30 * 60 * 1000) { // 30 minutes - return getScheduleData().then((meetingpointData_) => { - lastUpdate = new Date(); - meetingpointData = meetingpointData_; - - return meetingpointData; - }); - } - - return Promise.resolve(meetingpointData); -} - -// Debounce getScheduleDataCacheWrapper. This ensures that no requests will be -// waited if a user requests the schedule data while the schedule data is -// already being requested by another user. -module.exports = debounce(getScheduleDataCacheWrapper); diff --git a/src/server/lib/schools/hetmml/getURLOfUser.js b/src/server/lib/schools/hetmml/getURLOfUser.js deleted file mode 100644 index 5010207..0000000 --- a/src/server/lib/schools/hetmml/getURLOfUser.js +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright (C) 2018 Noah Loomans - * - * This file is part of rooster.hetmml.nl. - * - * rooster.hetmml.nl is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * rooster.hetmml.nl is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with rooster.hetmml.nl. If not, see . - * - */ - -const leftPad = require('left-pad'); // I imported this just to piss you off ;) - -function getURLOfUser(scheduleType, type, index, week) { - return `/${scheduleType}roosters/${leftPad(week, 2, '0')}/${type}/${type}${leftPad(index + 1, 5, '0')}.htm`; -} - -module.exports = getURLOfUser; diff --git a/src/server/lib/schools/hetmml/parseSchedule.js b/src/server/lib/schools/hetmml/parseSchedule.js deleted file mode 100644 index 066d637..0000000 --- a/src/server/lib/schools/hetmml/parseSchedule.js +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright (C) 2018 Noah Loomans - * - * This file is part of rooster.hetmml.nl. - * - * rooster.hetmml.nl is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * rooster.hetmml.nl is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with rooster.hetmml.nl. If not, see . - * - */ - -const { JSDOM } = require('jsdom'); - -function fixFirstLargeScheduleItem(trNodeList) { - return Array.from(trNodeList).some((trNode, timeOfDay) => { - const tdNodeList = trNode.children; - - return Array.from(tdNodeList).some((tdNode, dayOfWeek) => { - const height = tdNode.rowSpan / 2; - if (height === 1) { - return false; - } - - tdNode.rowSpan = 2; // eslint-disable-line no-param-reassign - - for (let i = 1; i < height; i += 1) { - // Are we at the end of the table? - if (dayOfWeek === 4) { - // If so, we cannot use insertBefore, because the is no node to insert - // it before. Use appendChild instead. - trNodeList[timeOfDay + i].appendChild(tdNode.cloneNode(true)); - } else { - trNodeList[timeOfDay + i] - .insertBefore( - tdNode.cloneNode(true), - trNodeList[timeOfDay + i].children[dayOfWeek], - ); - } - } - - return true; - }); - }); -} - -function parseSchedule(axiosResponse) { - const dom = new JSDOM(axiosResponse.data); - const { window } = dom; - const { document } = window; - - const tableNode = document.querySelector('center > table'); - const tbodyNode = tableNode.querySelector('tbody'); - const trNodeList = tbodyNode.children; - - Array.from(trNodeList).forEach((trNode, timeOfDay) => { - const tdNodeList = trNode.children; - - if (timeOfDay === 0 || trNode.children.length === 0) { - tbodyNode.removeChild(trNode); - return; - } - - Array.from(tdNodeList).forEach((tdNode, dayOfWeek) => { - if (dayOfWeek === 0) { - trNode.removeChild(tdNode); - } - }); - }); - - let shouldContinue = true; - while (shouldContinue) { - shouldContinue = fixFirstLargeScheduleItem(trNodeList); - } - - const scheduleItems = []; - - Array.from(trNodeList).forEach((trNode, timeOfDay) => { - const tdNodeList = trNode.children; - Array.from(tdNodeList).forEach((tdNode, dayOfWeek) => { - if (tdNode.textContent.trim() === '') { - return; - } - - const childTableNode = tdNode.querySelector('table'); - const childTrNodeList = childTableNode.querySelectorAll('tr'); - - Array.from(childTrNodeList).forEach((childTrNode) => { - const subject = childTrNode.children[0].textContent.trim(); - const attendees = childTrNode.children[1] - ? childTrNode.children[1].textContent.trim() - : undefined; - const location = childTrNode.children[2] - ? childTrNode.children[2].textContent.trim() - : undefined; - - scheduleItems.push({ - startTime: timeOfDay, - endTime: timeOfDay + 1, - dayOfWeek, - subject, - attendees, - location, - }); - }); - }); - }); - - return scheduleItems; -} - -module.exports = parseSchedule; diff --git a/src/server/routes/getSchedule.js b/src/server/routes/getSchedule.js index 2ade31a..fb78e97 100644 --- a/src/server/routes/getSchedule.js +++ b/src/server/routes/getSchedule.js @@ -22,10 +22,10 @@ const express = require('express'); const router = express.Router(); -const getScheduleData = require('../lib/schools/hetmml/getScheduleData'); -const getURLOfUser = require('../lib/schools/hetmml/getURLOfUser'); -const axios = require('../lib/schools/hetmml/axios'); -const parseSchedule = require('../lib/schools/hetmml/parseSchedule'); +const { getUsers } = require('../../shared/lib/db'); +const getURLOfUser = require('../../shared/lib/getURLOfUser'); +const axios = require('../../shared/lib/axios'); +const parseSchedule = require('../lib/parseSchedule'); // copied from http://www.meetingpointmco.nl/Roosters-AL/doc/dagroosters/untisscripts.js, // were using the same code as they do to be sure that we always get the same @@ -46,14 +46,14 @@ function currentWeekNumber() { return 1 + Math.ceil((firstThursday - target) / 604800000); } -async function getSchedule(userType, userValue, week, scheduleType = 'dag') { - const { users } = await getScheduleData(); +async function getSchedule(userType, userName, week, scheduleType = 'dag') { + const users = await getUsers(); const user = users.filter(user_ => ( - user_.type === userType && user_.value === userValue + user_.type === userType && user_.name === userName ))[0]; if (!user) { - throw new Error(`${userType}/${userValue} is not in the user index.`); + throw new Error(`${userType}/${userName} is not in the user index.`); } if (!week) { diff --git a/src/server/routes/index.js b/src/server/routes/index.js index d647210..b0e86ee 100644 --- a/src/server/routes/index.js +++ b/src/server/routes/index.js @@ -21,31 +21,13 @@ const express = require('express'); const router = express.Router(); -const getScheduleData = require('../lib/schools/hetmml/getScheduleData'); +const { getUsers } = require('../../shared/lib/db'); /* GET home page. */ -router.get(['/', '/s/*', '/t/*', '/r/*', '/c/*'], (req, res) => { - getScheduleData().then(({ users, dailyScheduleWeeks, basisScheduleWeeks }) => { - const isBeta = process.env.BETA === '1'; - - const flags = []; - if (isBeta) { - flags.push('BETA'); - flags.push('NO_FEATURE_DETECT'); - } else if (req.query.nfd != null) { - flags.push('NO_FEATURE_DETECT'); - } - - const flagsStr = `var FLAGS = ${JSON.stringify(flags)};`; - const usersStr = `var USERS = ${JSON.stringify(users)};`; - const dailyScheduleWeeksStr = `var DAGROOSTER_WEEKS = ${JSON.stringify(dailyScheduleWeeks)}`; - const basisScheduleWeeksStr = `var BASISROOSTER_WEEKS = ${JSON.stringify(basisScheduleWeeks)}`; - +router.get(['/', '/student/*', '/teacher/*', '/room/*', '/class/*'], (req, res) => { + getUsers().then((users) => { res.render('index', { - flagsStr, - usersStr, - dailyScheduleWeeksStr, - basisScheduleWeeksStr, + usersStr: JSON.stringify(users), }); }); // .catch(() => { diff --git a/src/shared/lib/axios.js b/src/shared/lib/axios.js new file mode 100644 index 0000000..2de6047 --- /dev/null +++ b/src/shared/lib/axios.js @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const axios = require('axios'); +const iconv = require('iconv-lite'); + +/** + * Create a new axios instance with sensible defaults for connecting to + * meetingpointmco.nl +*/ +const instance = axios.create({ + baseURL: 'http://www.meetingpointmco.nl/Roosters-AL/doc/', + timeout: 5000, + responseType: 'arraybuffer', + transformResponse: [responseBody => iconv.decode(responseBody, 'iso-8859-1')], +}); + +module.exports = instance; diff --git a/src/shared/lib/db.js b/src/shared/lib/db.js new file mode 100644 index 0000000..cb0be09 --- /dev/null +++ b/src/shared/lib/db.js @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const { Pool } = require('pg'); + +const pool = new Pool({ + database: process.env.PGDATABASE || 'rooster_hetmml_nl', +}); + +// the pool with emit an error on behalf of any idle clients +// it contains if a backend error or network partition happens +pool.on('error', (error) => { + console.error('Unexpected error on idle client', error); + // TODO: Do we want to exit here? + process.exit(-1); +}); + +async function insertUsers(users) { + await pool.query('TRUNCATE TABLE schedule_user'); + const promises = users.map(user => ( + pool.query( + 'INSERT INTO schedule_user(key, type, name, alt_name, index) VALUES ($1, $2, $3, $4, $5)', + [user.key, user.type, user.name, user.altName, user.index], + ) + )); + await Promise.all(promises); +} + +async function getUsers() { + const { rows } = await pool.query( + 'SELECT key, type, name, alt_name, index FROM schedule_user', + ); + + const users = rows.map(row => ({ + key: row.key, + type: row.type, + name: row.name, + altName: row.altName, + index: row.index, + })); + + return users; +} + +module.exports = { insertUsers, getUsers }; diff --git a/src/shared/lib/getURLOfUser.js b/src/shared/lib/getURLOfUser.js new file mode 100644 index 0000000..934f235 --- /dev/null +++ b/src/shared/lib/getURLOfUser.js @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const leftPad = require('left-pad'); // I imported this just to piss you off ;) + +const shortTypeOf = { + class: 'c', + teacher: 't', + student: 's', + room: 'r', +}; + +function getURLOfUser(scheduleType, longType, index, week) { + const type = shortTypeOf[longType]; + return `/${scheduleType}roosters/${leftPad(week, 2, '0')}/${type}/${type}${leftPad(index + 1, 5, '0')}.htm`; +} + +module.exports = getURLOfUser; diff --git a/src/sync/index.js b/src/sync/index.js new file mode 100644 index 0000000..3ccbad9 --- /dev/null +++ b/src/sync/index.js @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const moment = require('moment'); + +const axios = require('../shared/lib/axios'); +const { insertUsers } = require('../shared/lib/db'); +const getUrlOfUser = require('../shared/lib/getURLOfUser'); +const { scrapeUsers, scrapeAltName } = require('./scrapeScheduleData'); + +moment.locale('nl'); + +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); + +async function fetchUsers() { + const navBar = await axios.get('/dagroosters/frames/navbar.htm'); + const users = scrapeUsers(navBar.data); + + return users; +} + +async function fetchAltNameOfUser(user) { + const url = getUrlOfUser('dag', user.type, user.index, moment().week()); + const schedule = await axios.get(url); + const altName = scrapeAltName(schedule.data); + return altName; +} + +function simplifyNameOfUser(user) { + if (user.type === 'class') { + return { + ...user, + name: user.name.slice(2), + }; + } + + return user; +} + +async function sync() { + const users = (await fetchUsers()).map(simplifyNameOfUser); + + const alts = []; + + /* eslint-disable no-restricted-syntax, no-await-in-loop */ + const teachers = users.filter(user => user.type === 'teacher') + .slice(0, 3); // TODO: Remove this. + for (const teacher of teachers) { + await delay(500); + const altName = await fetchAltNameOfUser(teacher); + console.log(`${teacher.key}: ${altName}`); + if (altName) { + alts.push({ key: teacher.key, altName }); + } + } + /* eslint-enable no-restricted-syntax, no-await-in-loop */ + + const usersWithAlts = users.map((user) => { + const { altName } = ( + alts.find(altUser => altUser.key === user.key) || { altName: undefined } + ); + + return { + ...user, + altName, + }; + }); + + await insertUsers(usersWithAlts); + console.log(usersWithAlts.filter(user => user.type === 'teacher')); +} + +sync() + .then(() => { + process.exit(); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/src/sync/scrapeScheduleData.js b/src/sync/scrapeScheduleData.js new file mode 100644 index 0000000..fc7193a --- /dev/null +++ b/src/sync/scrapeScheduleData.js @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const cheerio = require('cheerio'); +const _ = require('lodash'); + +/** + * Scrape all the valid users from a meetingpoint navbar. + * @param {string} html The html of a meetingpoint navbar. + * @returns {*} + * [ + * { type: 't', value: 'akh', index: 0 }, + * ... + * { type: 's', value: '18561', index: 245 }, + * ... + * { type: 'r', value: '008-mk', index: 2 }, + * ... + * { type: 'c', value: '6-5H2', index: 23 }, + * ... + * ] + */ +function scrapeUsers(html) { + const page = cheerio.load(html); + const script = page('script').eq(1).text(); + + const regexs = [/var classes = \[(.+)\];/, /var teachers = \[(.+)\];/, /var rooms = \[(.+)\];/, /var students = \[(.+)\];/]; + const items = regexs.map(regex => script.match(regex)[1].split(',').map(item => item.replace(/"/g, ''))); + + const classes = items[0].map((name, index) => ({ + key: `c/${name}`, + type: 'class', + name, + index, + })); + + const teachers = items[1].map((name, index) => ({ + key: `t/${name}`, + type: 'teacher', + name, + index, + })); + + const rooms = items[2].map((name, index) => ({ + key: `r/${name}`, + type: 'room', + name, + index, + })); + + const students = items[3].map((name, index) => ({ + key: `s/${name}`, + type: 'student', + name, + index, + })); + + return _.flatten([classes, teachers, rooms, students]); +} + +/** + * scrape the alt text (the text next to the short code) from a + * specific meetingpoint schedule. + * @param {string} html The html of a specific meetingpoint schedule. + * @returns {string} + */ +function scrapeAltName(html) { + const page = cheerio.load(html); + return page('center > font').eq(2).text().trim() || undefined; +} + +module.exports.scrapeUsers = scrapeUsers; +module.exports.scrapeAltName = scrapeAltName; -- cgit v1.1