aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/client/.eslintrc5
-rw-r--r--src/client/react/components/container/HelpBox.js2
-rw-r--r--src/client/views/index.handlebars5
-rw-r--r--src/server/lib/parseSchedule.js (renamed from src/server/lib/schools/hetmml/parseSchedule.js)0
-rw-r--r--src/server/lib/schools/hetmml/getScheduleData.js206
-rw-r--r--src/server/routes/getSchedule.js16
-rw-r--r--src/server/routes/index.js26
-rw-r--r--src/shared/lib/axios.js (renamed from src/server/lib/schools/hetmml/axios.js)0
-rw-r--r--src/shared/lib/db.js62
-rw-r--r--src/shared/lib/getURLOfUser.js (renamed from src/server/lib/schools/hetmml/getURLOfUser.js)10
-rw-r--r--src/sync/index.js97
-rw-r--r--src/sync/scrapeScheduleData.js89
12 files changed, 277 insertions, 241 deletions
diff --git a/src/client/.eslintrc b/src/client/.eslintrc
new file mode 100644
index 0000000..6479217
--- /dev/null
+++ b/src/client/.eslintrc
@@ -0,0 +1,5 @@
+{
+ "rules": {
+ "no-console": "warn"
+ }
+}
diff --git a/src/client/react/components/container/HelpBox.js b/src/client/react/components/container/HelpBox.js
index d10c26b..f4fbbe9 100644
--- a/src/client/react/components/container/HelpBox.js
+++ b/src/client/react/components/container/HelpBox.js
@@ -21,6 +21,8 @@
import { connect } from 'react-redux';
import HelpBox from '../presentational/HelpBox';
+console.log('hi');
+
const mapStateToProps = state => ({
isVisible: !state.search || (state.search.results.length === 0 && state.search.text === ''),
});
diff --git a/src/client/views/index.handlebars b/src/client/views/index.handlebars
index 5105439..93eccf1 100644
--- a/src/client/views/index.handlebars
+++ b/src/client/views/index.handlebars
@@ -28,10 +28,7 @@
<body>
<div id="root"></div>
<script>
- {{{ flagsStr }}}
- {{{ usersStr }}}
- {{{ dailyScheduleWeeksStr }}}
- {{{ basisScheduleWeeksStr }}}
+ var USERS = {{{ usersStr }}};
</script>
<script src="/bundle.js"></script>
</body>
diff --git a/src/server/lib/schools/hetmml/parseSchedule.js b/src/server/lib/parseSchedule.js
index 066d637..066d637 100644
--- a/src/server/lib/schools/hetmml/parseSchedule.js
+++ b/src/server/lib/parseSchedule.js
diff --git a/src/server/lib/schools/hetmml/getScheduleData.js b/src/server/lib/schools/hetmml/getScheduleData.js
deleted file mode 100644
index ead856f..0000000
--- a/src/server/lib/schools/hetmml/getScheduleData.js
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Copyright (C) 2018 Noah Loomans
- *
- * This file is part of rooster.hetmml.nl.
- *
- * rooster.hetmml.nl is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * rooster.hetmml.nl is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with rooster.hetmml.nl. If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-const cheerio = require('cheerio');
-const debounce = require('promise-debounce');
-const _ = require('lodash');
-
-const getUrlOfUser = require('./getURLOfUser');
-const axios = require('./axios');
-
-let meetingpointData;
-let lastUpdate;
-
-/**
- * Scrape all the valid users from a meetingpoint navbar.
- * @param {string} html The html of a meetingpoint navbar.
- * @returns {*}
- * [
- * { type: 't', value: 'akh', index: 0 },
- * ...
- * { type: 's', value: '18561', index: 245 },
- * ...
- * { type: 'r', value: '008-mk', index: 2 },
- * ...
- * { type: 'c', value: '6-5H2', index: 23 },
- * ...
- * ]
- */
-function scrapeUsers(html) {
- const page = cheerio.load(html);
- const script = page('script').eq(1).text();
-
- const regexs = [/var classes = \[(.+)\];/, /var teachers = \[(.+)\];/, /var rooms = \[(.+)\];/, /var students = \[(.+)\];/];
- const items = regexs.map(regex => script.match(regex)[1].split(',').map(item => item.replace(/"/g, '')));
-
- const classes = items[0].map((item, index) => ({
- type: 'c',
- value: item,
- index,
- }));
-
- const teachers = items[1].map((item, index) => ({
- type: 't',
- value: item,
- index,
- }));
-
- const rooms = items[2].map((item, index) => ({
- type: 'r',
- value: item,
- index,
- }));
-
- const students = items[3].map((item, index) => ({
- type: 's',
- value: item,
- index,
- }));
-
- return _.flatten([classes, teachers, rooms, students]);
-}
-
-/**
- * Scrape the known valid weeks from a meetingpoint navbar.
- *
- * There probably are more valid weeks, but these once are garanteed to be
- * valid.
- * @param {string} html The html of a meetingpoint navbar.
- * @returns {*} [{ id: string, text: string }, ...]
- */
-function scrapeWeeks(html) {
- const page = cheerio.load(html);
- const weekSelector = page('select[name="week"]');
- const weeks = _.map(weekSelector.children(), option => ({
- id: cheerio(option).attr('value'),
- text: cheerio(option).text(),
- }));
-
- return weeks;
-}
-
-/**
- * scrape the alt text (the text next to the short code) from a
- * specific meetingpoint schedule.
- * @param {string} html The html of a specific meetingpoint schedule.
- * @returns {string}
- */
-function scrapeAltText(html) {
- const page = cheerio.load(html);
- return page('center > font').eq(2).text().trim();
-}
-
-/**
- * Combines two user array, if a dublicate user is present, the first one will
- * be used.
- *
- * This function is currently used to merge a subset of users with alts
- * attached to them with a compleat set of users without alts.
- * @param {*} usersArrays An array of user arrays.
- */
-function combineUsers(usersArrays) {
- return _.uniqBy(_.flatten(usersArrays), user => `${user.type}/${user.value}`);
-}
-
-/**
- * Requests and adds an alt field to the given users.
- *
- * For example, it will add the teacher name to a teacher object.
- *
- * @param {*} users [{ type: string, value: string, index: number }, ...]
- * @returns {*} [{ type: string, value: string, alt: string, index: number }, ...]
- */
-function getAlts(users) {
- const requests = users.map(user => (
- axios.get(
- getUrlOfUser('dag', user.type, user.index, 7),
- { timeout: 8000 },
- )
- ));
-
- return Promise.all(requests).then(teacherResponses => (
- teacherResponses.map((teacherResponse, index) => {
- const teacherName = scrapeAltText(teacherResponse.data);
-
- return {
- ...users[index],
- alt: teacherName,
- };
- })
- ));
-}
-
-/**
- * Requests all the relevent data from the meetingpoint server
- * This is very expensive! Only call when you absolutely need to.
- * @returns {Promise} { users, dailyScheduleWeeks, basisScheduleWeeks }
- */
-function getScheduleData() {
- const navbarRequests = [
- axios.get('/dagroosters/frames/navbar.htm'),
- axios.get('/basisroosters/frames/navbar.htm'),
- ];
-
- return Promise.all(navbarRequests)
- .then(([dailyScheduleResponse, basisScheduleResponse]) => {
- const users = scrapeUsers(dailyScheduleResponse.data);
- const dailyScheduleWeeks = scrapeWeeks(dailyScheduleResponse.data);
- const basisScheduleWeeks = scrapeWeeks(basisScheduleResponse.data);
-
- const teachers = users.filter(user => user.type === 't');
-
- return getAlts(teachers)
- .then(teachersWithAlts => ({
- users: combineUsers([teachersWithAlts, users]),
- dailyScheduleWeeks,
- basisScheduleWeeks,
- }))
- .catch(() => ({
- // Just return the user data without the alts if getAlts fails, since
- // the alts are non-essential.
- users,
- dailyScheduleWeeks,
- basisScheduleWeeks,
- }));
- });
-}
-
-/**
- * Wrapper around getScheduleData that is cheap to call. In most cases it
- * returns a cached version. The cache is stored for 30 minutes.
- * @returns {Promise} { users, dailyScheduleWeeks, basisScheduleWeeks }
-*/
-function getScheduleDataCacheWrapper() {
- if (meetingpointData == null || new Date() - lastUpdate > 30 * 60 * 1000) { // 30 minutes
- return getScheduleData().then((meetingpointData_) => {
- lastUpdate = new Date();
- meetingpointData = meetingpointData_;
-
- return meetingpointData;
- });
- }
-
- return Promise.resolve(meetingpointData);
-}
-
-// Debounce getScheduleDataCacheWrapper. This ensures that no requests will be
-// waited if a user requests the schedule data while the schedule data is
-// already being requested by another user.
-module.exports = debounce(getScheduleDataCacheWrapper);
diff --git a/src/server/routes/getSchedule.js b/src/server/routes/getSchedule.js
index 2ade31a..fb78e97 100644
--- a/src/server/routes/getSchedule.js
+++ b/src/server/routes/getSchedule.js
@@ -22,10 +22,10 @@ const express = require('express');
const router = express.Router();
-const getScheduleData = require('../lib/schools/hetmml/getScheduleData');
-const getURLOfUser = require('../lib/schools/hetmml/getURLOfUser');
-const axios = require('../lib/schools/hetmml/axios');
-const parseSchedule = require('../lib/schools/hetmml/parseSchedule');
+const { getUsers } = require('../../shared/lib/db');
+const getURLOfUser = require('../../shared/lib/getURLOfUser');
+const axios = require('../../shared/lib/axios');
+const parseSchedule = require('../lib/parseSchedule');
// copied from http://www.meetingpointmco.nl/Roosters-AL/doc/dagroosters/untisscripts.js,
// were using the same code as they do to be sure that we always get the same
@@ -46,14 +46,14 @@ function currentWeekNumber() {
return 1 + Math.ceil((firstThursday - target) / 604800000);
}
-async function getSchedule(userType, userValue, week, scheduleType = 'dag') {
- const { users } = await getScheduleData();
+async function getSchedule(userType, userName, week, scheduleType = 'dag') {
+ const users = await getUsers();
const user = users.filter(user_ => (
- user_.type === userType && user_.value === userValue
+ user_.type === userType && user_.name === userName
))[0];
if (!user) {
- throw new Error(`${userType}/${userValue} is not in the user index.`);
+ throw new Error(`${userType}/${userName} is not in the user index.`);
}
if (!week) {
diff --git a/src/server/routes/index.js b/src/server/routes/index.js
index d647210..b0e86ee 100644
--- a/src/server/routes/index.js
+++ b/src/server/routes/index.js
@@ -21,31 +21,13 @@
const express = require('express');
const router = express.Router();
-const getScheduleData = require('../lib/schools/hetmml/getScheduleData');
+const { getUsers } = require('../../shared/lib/db');
/* GET home page. */
-router.get(['/', '/s/*', '/t/*', '/r/*', '/c/*'], (req, res) => {
- getScheduleData().then(({ users, dailyScheduleWeeks, basisScheduleWeeks }) => {
- const isBeta = process.env.BETA === '1';
-
- const flags = [];
- if (isBeta) {
- flags.push('BETA');
- flags.push('NO_FEATURE_DETECT');
- } else if (req.query.nfd != null) {
- flags.push('NO_FEATURE_DETECT');
- }
-
- const flagsStr = `var FLAGS = ${JSON.stringify(flags)};`;
- const usersStr = `var USERS = ${JSON.stringify(users)};`;
- const dailyScheduleWeeksStr = `var DAGROOSTER_WEEKS = ${JSON.stringify(dailyScheduleWeeks)}`;
- const basisScheduleWeeksStr = `var BASISROOSTER_WEEKS = ${JSON.stringify(basisScheduleWeeks)}`;
-
+router.get(['/', '/student/*', '/teacher/*', '/room/*', '/class/*'], (req, res) => {
+ getUsers().then((users) => {
res.render('index', {
- flagsStr,
- usersStr,
- dailyScheduleWeeksStr,
- basisScheduleWeeksStr,
+ usersStr: JSON.stringify(users),
});
});
// .catch(() => {
diff --git a/src/server/lib/schools/hetmml/axios.js b/src/shared/lib/axios.js
index 2de6047..2de6047 100644
--- a/src/server/lib/schools/hetmml/axios.js
+++ b/src/shared/lib/axios.js
diff --git a/src/shared/lib/db.js b/src/shared/lib/db.js
new file mode 100644
index 0000000..cb0be09
--- /dev/null
+++ b/src/shared/lib/db.js
@@ -0,0 +1,62 @@
+/**
+ * Copyright (C) 2018 Noah Loomans
+ *
+ * This file is part of rooster.hetmml.nl.
+ *
+ * rooster.hetmml.nl is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * rooster.hetmml.nl is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with rooster.hetmml.nl. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const { Pool } = require('pg');
+
+const pool = new Pool({
+ database: process.env.PGDATABASE || 'rooster_hetmml_nl',
+});
+
+// the pool with emit an error on behalf of any idle clients
+// it contains if a backend error or network partition happens
+pool.on('error', (error) => {
+ console.error('Unexpected error on idle client', error);
+ // TODO: Do we want to exit here?
+ process.exit(-1);
+});
+
+async function insertUsers(users) {
+ await pool.query('TRUNCATE TABLE schedule_user');
+ const promises = users.map(user => (
+ pool.query(
+ 'INSERT INTO schedule_user(key, type, name, alt_name, index) VALUES ($1, $2, $3, $4, $5)',
+ [user.key, user.type, user.name, user.altName, user.index],
+ )
+ ));
+ await Promise.all(promises);
+}
+
+async function getUsers() {
+ const { rows } = await pool.query(
+ 'SELECT key, type, name, alt_name, index FROM schedule_user',
+ );
+
+ const users = rows.map(row => ({
+ key: row.key,
+ type: row.type,
+ name: row.name,
+ altName: row.altName,
+ index: row.index,
+ }));
+
+ return users;
+}
+
+module.exports = { insertUsers, getUsers };
diff --git a/src/server/lib/schools/hetmml/getURLOfUser.js b/src/shared/lib/getURLOfUser.js
index 5010207..934f235 100644
--- a/src/server/lib/schools/hetmml/getURLOfUser.js
+++ b/src/shared/lib/getURLOfUser.js
@@ -20,7 +20,15 @@
const leftPad = require('left-pad'); // I imported this just to piss you off ;)
-function getURLOfUser(scheduleType, type, index, week) {
+const shortTypeOf = {
+ class: 'c',
+ teacher: 't',
+ student: 's',
+ room: 'r',
+};
+
+function getURLOfUser(scheduleType, longType, index, week) {
+ const type = shortTypeOf[longType];
return `/${scheduleType}roosters/${leftPad(week, 2, '0')}/${type}/${type}${leftPad(index + 1, 5, '0')}.htm`;
}
diff --git a/src/sync/index.js b/src/sync/index.js
new file mode 100644
index 0000000..3ccbad9
--- /dev/null
+++ b/src/sync/index.js
@@ -0,0 +1,97 @@
+/**
+ * Copyright (C) 2018 Noah Loomans
+ *
+ * This file is part of rooster.hetmml.nl.
+ *
+ * rooster.hetmml.nl is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * rooster.hetmml.nl is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with rooster.hetmml.nl. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const moment = require('moment');
+
+const axios = require('../shared/lib/axios');
+const { insertUsers } = require('../shared/lib/db');
+const getUrlOfUser = require('../shared/lib/getURLOfUser');
+const { scrapeUsers, scrapeAltName } = require('./scrapeScheduleData');
+
+moment.locale('nl');
+
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+async function fetchUsers() {
+ const navBar = await axios.get('/dagroosters/frames/navbar.htm');
+ const users = scrapeUsers(navBar.data);
+
+ return users;
+}
+
+async function fetchAltNameOfUser(user) {
+ const url = getUrlOfUser('dag', user.type, user.index, moment().week());
+ const schedule = await axios.get(url);
+ const altName = scrapeAltName(schedule.data);
+ return altName;
+}
+
+function simplifyNameOfUser(user) {
+ if (user.type === 'class') {
+ return {
+ ...user,
+ name: user.name.slice(2),
+ };
+ }
+
+ return user;
+}
+
+async function sync() {
+ const users = (await fetchUsers()).map(simplifyNameOfUser);
+
+ const alts = [];
+
+ /* eslint-disable no-restricted-syntax, no-await-in-loop */
+ const teachers = users.filter(user => user.type === 'teacher')
+ .slice(0, 3); // TODO: Remove this.
+ for (const teacher of teachers) {
+ await delay(500);
+ const altName = await fetchAltNameOfUser(teacher);
+ console.log(`${teacher.key}: ${altName}`);
+ if (altName) {
+ alts.push({ key: teacher.key, altName });
+ }
+ }
+ /* eslint-enable no-restricted-syntax, no-await-in-loop */
+
+ const usersWithAlts = users.map((user) => {
+ const { altName } = (
+ alts.find(altUser => altUser.key === user.key) || { altName: undefined }
+ );
+
+ return {
+ ...user,
+ altName,
+ };
+ });
+
+ await insertUsers(usersWithAlts);
+ console.log(usersWithAlts.filter(user => user.type === 'teacher'));
+}
+
+sync()
+ .then(() => {
+ process.exit();
+ })
+ .catch((error) => {
+ console.error(error);
+ process.exit(1);
+ });
diff --git a/src/sync/scrapeScheduleData.js b/src/sync/scrapeScheduleData.js
new file mode 100644
index 0000000..fc7193a
--- /dev/null
+++ b/src/sync/scrapeScheduleData.js
@@ -0,0 +1,89 @@
+/**
+ * Copyright (C) 2018 Noah Loomans
+ *
+ * This file is part of rooster.hetmml.nl.
+ *
+ * rooster.hetmml.nl is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * rooster.hetmml.nl is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with rooster.hetmml.nl. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const cheerio = require('cheerio');
+const _ = require('lodash');
+
+/**
+ * Scrape all the valid users from a meetingpoint navbar.
+ * @param {string} html The html of a meetingpoint navbar.
+ * @returns {*}
+ * [
+ * { type: 't', value: 'akh', index: 0 },
+ * ...
+ * { type: 's', value: '18561', index: 245 },
+ * ...
+ * { type: 'r', value: '008-mk', index: 2 },
+ * ...
+ * { type: 'c', value: '6-5H2', index: 23 },
+ * ...
+ * ]
+ */
+function scrapeUsers(html) {
+ const page = cheerio.load(html);
+ const script = page('script').eq(1).text();
+
+ const regexs = [/var classes = \[(.+)\];/, /var teachers = \[(.+)\];/, /var rooms = \[(.+)\];/, /var students = \[(.+)\];/];
+ const items = regexs.map(regex => script.match(regex)[1].split(',').map(item => item.replace(/"/g, '')));
+
+ const classes = items[0].map((name, index) => ({
+ key: `c/${name}`,
+ type: 'class',
+ name,
+ index,
+ }));
+
+ const teachers = items[1].map((name, index) => ({
+ key: `t/${name}`,
+ type: 'teacher',
+ name,
+ index,
+ }));
+
+ const rooms = items[2].map((name, index) => ({
+ key: `r/${name}`,
+ type: 'room',
+ name,
+ index,
+ }));
+
+ const students = items[3].map((name, index) => ({
+ key: `s/${name}`,
+ type: 'student',
+ name,
+ index,
+ }));
+
+ return _.flatten([classes, teachers, rooms, students]);
+}
+
+/**
+ * scrape the alt text (the text next to the short code) from a
+ * specific meetingpoint schedule.
+ * @param {string} html The html of a specific meetingpoint schedule.
+ * @returns {string}
+ */
+function scrapeAltName(html) {
+ const page = cheerio.load(html);
+ return page('center > font').eq(2).text().trim() || undefined;
+}
+
+module.exports.scrapeUsers = scrapeUsers;
+module.exports.scrapeAltName = scrapeAltName;