From 5c265c04ad513d845a41c7866c3ed231c8d5e68e Mon Sep 17 00:00:00 2001 From: Noah Loomans Date: Sat, 7 Jul 2018 17:34:02 +0200 Subject: server: Use database for users --- src/sync/index.js | 97 ++++++++++++++++++++++++++++++++++++++++++ src/sync/scrapeScheduleData.js | 89 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 src/sync/index.js create mode 100644 src/sync/scrapeScheduleData.js (limited to 'src/sync') diff --git a/src/sync/index.js b/src/sync/index.js new file mode 100644 index 0000000..3ccbad9 --- /dev/null +++ b/src/sync/index.js @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const moment = require('moment'); + +const axios = require('../shared/lib/axios'); +const { insertUsers } = require('../shared/lib/db'); +const getUrlOfUser = require('../shared/lib/getURLOfUser'); +const { scrapeUsers, scrapeAltName } = require('./scrapeScheduleData'); + +moment.locale('nl'); + +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); + +async function fetchUsers() { + const navBar = await axios.get('/dagroosters/frames/navbar.htm'); + const users = scrapeUsers(navBar.data); + + return users; +} + +async function fetchAltNameOfUser(user) { + const url = getUrlOfUser('dag', user.type, user.index, moment().week()); + const schedule = await axios.get(url); + const altName = scrapeAltName(schedule.data); + return altName; +} + +function simplifyNameOfUser(user) { + if (user.type === 'class') { + return { + ...user, + name: user.name.slice(2), + }; + } + + return user; +} + +async function sync() { + const users = (await fetchUsers()).map(simplifyNameOfUser); + + const alts = []; + + /* eslint-disable no-restricted-syntax, no-await-in-loop */ + const teachers = users.filter(user => user.type === 'teacher') + .slice(0, 3); // TODO: Remove this. + for (const teacher of teachers) { + await delay(500); + const altName = await fetchAltNameOfUser(teacher); + console.log(`${teacher.key}: ${altName}`); + if (altName) { + alts.push({ key: teacher.key, altName }); + } + } + /* eslint-enable no-restricted-syntax, no-await-in-loop */ + + const usersWithAlts = users.map((user) => { + const { altName } = ( + alts.find(altUser => altUser.key === user.key) || { altName: undefined } + ); + + return { + ...user, + altName, + }; + }); + + await insertUsers(usersWithAlts); + console.log(usersWithAlts.filter(user => user.type === 'teacher')); +} + +sync() + .then(() => { + process.exit(); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/src/sync/scrapeScheduleData.js b/src/sync/scrapeScheduleData.js new file mode 100644 index 0000000..fc7193a --- /dev/null +++ b/src/sync/scrapeScheduleData.js @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2018 Noah Loomans + * + * This file is part of rooster.hetmml.nl. + * + * rooster.hetmml.nl is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rooster.hetmml.nl is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with rooster.hetmml.nl. If not, see . + * + */ + +const cheerio = require('cheerio'); +const _ = require('lodash'); + +/** + * Scrape all the valid users from a meetingpoint navbar. + * @param {string} html The html of a meetingpoint navbar. + * @returns {*} + * [ + * { type: 't', value: 'akh', index: 0 }, + * ... + * { type: 's', value: '18561', index: 245 }, + * ... + * { type: 'r', value: '008-mk', index: 2 }, + * ... + * { type: 'c', value: '6-5H2', index: 23 }, + * ... + * ] + */ +function scrapeUsers(html) { + const page = cheerio.load(html); + const script = page('script').eq(1).text(); + + const regexs = [/var classes = \[(.+)\];/, /var teachers = \[(.+)\];/, /var rooms = \[(.+)\];/, /var students = \[(.+)\];/]; + const items = regexs.map(regex => script.match(regex)[1].split(',').map(item => item.replace(/"/g, ''))); + + const classes = items[0].map((name, index) => ({ + key: `c/${name}`, + type: 'class', + name, + index, + })); + + const teachers = items[1].map((name, index) => ({ + key: `t/${name}`, + type: 'teacher', + name, + index, + })); + + const rooms = items[2].map((name, index) => ({ + key: `r/${name}`, + type: 'room', + name, + index, + })); + + const students = items[3].map((name, index) => ({ + key: `s/${name}`, + type: 'student', + name, + index, + })); + + return _.flatten([classes, teachers, rooms, students]); +} + +/** + * scrape the alt text (the text next to the short code) from a + * specific meetingpoint schedule. + * @param {string} html The html of a specific meetingpoint schedule. + * @returns {string} + */ +function scrapeAltName(html) { + const page = cheerio.load(html); + return page('center > font').eq(2).text().trim() || undefined; +} + +module.exports.scrapeUsers = scrapeUsers; +module.exports.scrapeAltName = scrapeAltName; -- cgit v1.1