server: Use database for userssync-with-db

author: Noah Loomans <noahloomans@gmail.com> 2018-07-07 17:34:02 +0200
committer: Noah Loomans <noahloomans@gmail.com> 2018-07-07 17:34:02 +0200
commit: 5c265c04ad513d845a41c7866c3ed231c8d5e68e (patch)
tree: 9a86b66958e59e4a07d0c8e10731667b05990816 /src
parent: 8b1f29f2802a081a67151e47b9c52803d7cc568a (diff)
12 files changed, 277 insertions, 241 deletions
diff --git a/src/client/.eslintrc b/src/client/.eslintrc
new file mode 100644
index 0000000..6479217
--- /dev/null
+++ b/src/client/.eslintrc
@@ -0,0 +1,5 @@
+{
+  "rules": {
+    "no-console": "warn"
+  }
+}
diff --git a/src/client/react/components/container/HelpBox.js b/src/client/react/components/container/HelpBox.js
index d10c26b..f4fbbe9 100644
--- a/src/client/react/components/container/HelpBox.js
+++ b/src/client/react/components/container/HelpBox.js
@@ -21,6 +21,8 @@
 import { connect } from 'react-redux';
 import HelpBox from '../presentational/HelpBox';
 
+console.log('hi');
+
 const mapStateToProps = state => ({
   isVisible: !state.search || (state.search.results.length === 0 && state.search.text === ''),
 });
diff --git a/src/client/views/index.handlebars b/src/client/views/index.handlebars
index 5105439..93eccf1 100644
--- a/src/client/views/index.handlebars
+++ b/src/client/views/index.handlebars
@@ -28,10 +28,7 @@
 <body>
   <div id="root"></div>
   <script>
-    {{{ flagsStr }}}
-    {{{ usersStr }}}
-    {{{ dailyScheduleWeeksStr }}}
-    {{{ basisScheduleWeeksStr }}}
+    var USERS = {{{ usersStr }}};
   </script>
   <script src="/bundle.js"></script>
 </body>
diff --git a/src/server/lib/schools/hetmml/parseSchedule.js b/src/server/lib/parseSchedule.js
index 066d637..066d637 100644
--- a/src/server/lib/schools/hetmml/parseSchedule.js
+++ b/src/server/lib/parseSchedule.js
diff --git a/src/server/lib/schools/hetmml/getScheduleData.js b/src/server/lib/schools/hetmml/getScheduleData.js
deleted file mode 100644
index ead856f..0000000
--- a/src/server/lib/schools/hetmml/getScheduleData.js
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Copyright (C) 2018 Noah Loomans
- *
- * This file is part of rooster.hetmml.nl.
- *
- * rooster.hetmml.nl is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * rooster.hetmml.nl is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with rooster.hetmml.nl.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-const cheerio = require('cheerio');
-const debounce = require('promise-debounce');
-const _ = require('lodash');
-
-const getUrlOfUser = require('./getURLOfUser');
-const axios = require('./axios');
-
-let meetingpointData;
-let lastUpdate;
-
-/**
- * Scrape all the valid users from a meetingpoint navbar.
- * @param {string} html The html of a meetingpoint navbar.
- * @returns {*}
- * [
- *   { type: 't', value: 'akh', index: 0 },
- *   ...
- *   { type: 's', value: '18561', index: 245 },
- *   ...
- *   { type: 'r', value: '008-mk', index: 2 },
- *   ...
- *   { type: 'c', value: '6-5H2', index: 23 },
- *   ...
- * ]
- */
-function scrapeUsers(html) {
-  const page = cheerio.load(html);
-  const script = page('script').eq(1).text();
-
-  const regexs = [/var classes = \[(.+)\];/, /var teachers = \[(.+)\];/, /var rooms = \[(.+)\];/, /var students = \[(.+)\];/];
-  const items = regexs.map(regex => script.match(regex)[1].split(',').map(item => item.replace(/"/g, '')));
-
-  const classes = items[0].map((item, index) => ({
-    type: 'c',
-    value: item,
-    index,
-  }));
-
-  const teachers = items[1].map((item, index) => ({
-    type: 't',
-    value: item,
-    index,
-  }));
-
-  const rooms = items[2].map((item, index) => ({
-    type: 'r',
-    value: item,
-    index,
-  }));
-
-  const students = items[3].map((item, index) => ({
-    type: 's',
-    value: item,
-    index,
-  }));
-
-  return _.flatten([classes, teachers, rooms, students]);
-}
-
-/**
- * Scrape the known valid weeks from a meetingpoint navbar.
- *
- * There probably are more valid weeks, but these once are garanteed to be
- * valid.
- * @param {string} html The html of a meetingpoint navbar.
- * @returns {*} [{ id: string, text: string }, ...]
- */
-function scrapeWeeks(html) {
-  const page = cheerio.load(html);
-  const weekSelector = page('select[name="week"]');
-  const weeks = _.map(weekSelector.children(), option => ({
-    id: cheerio(option).attr('value'),
-    text: cheerio(option).text(),
-  }));
-
-  return weeks;
-}
-
-/**
- * scrape the alt text (the text next to the short code) from a
- * specific meetingpoint schedule.
- * @param {string} html The html of a specific meetingpoint schedule.
- * @returns {string}
- */
-function scrapeAltText(html) {
-  const page = cheerio.load(html);
-  return page('center > font').eq(2).text().trim();
-}
-
-/**
- * Combines two user array, if a dublicate user is present, the first one will
- * be used.
- *
- * This function is currently used to merge a subset of users with alts
- * attached to them with a compleat set of users without alts.
- * @param {*} usersArrays An array of user arrays.
- */
-function combineUsers(usersArrays) {
-  return _.uniqBy(_.flatten(usersArrays), user => `${user.type}/${user.value}`);
-}
-
-/**
- * Requests and adds an alt field to the given users.
- *
- * For example, it will add the teacher name to a teacher object.
- *
- * @param {*} users [{ type: string, value: string, index: number }, ...]
- * @returns {*} [{ type: string, value: string, alt: string, index: number }, ...]
- */
-function getAlts(users) {
-  const requests = users.map(user => (
-    axios.get(
-      getUrlOfUser('dag', user.type, user.index, 7),
-      { timeout: 8000 },
-    )
-  ));
-
-  return Promise.all(requests).then(teacherResponses => (
-    teacherResponses.map((teacherResponse, index) => {
-      const teacherName = scrapeAltText(teacherResponse.data);
-
-      return {
-        ...users[index],
-        alt: teacherName,
-      };
-    })
-  ));
-}
-
-/**
- * Requests all the relevent data from the meetingpoint server
- * This is very expensive! Only call when you absolutely need to.
- * @returns {Promise} { users, dailyScheduleWeeks, basisScheduleWeeks }
- */
-function getScheduleData() {
-  const navbarRequests = [
-    axios.get('/dagroosters/frames/navbar.htm'),
-    axios.get('/basisroosters/frames/navbar.htm'),
-  ];
-
-  return Promise.all(navbarRequests)
-    .then(([dailyScheduleResponse, basisScheduleResponse]) => {
-      const users = scrapeUsers(dailyScheduleResponse.data);
-      const dailyScheduleWeeks = scrapeWeeks(dailyScheduleResponse.data);
-      const basisScheduleWeeks = scrapeWeeks(basisScheduleResponse.data);
-
-      const teachers = users.filter(user => user.type === 't');
-
-      return getAlts(teachers)
-        .then(teachersWithAlts => ({
-          users: combineUsers([teachersWithAlts, users]),
-          dailyScheduleWeeks,
-          basisScheduleWeeks,
-        }))
-        .catch(() => ({
-          // Just return the user data without the alts if getAlts fails, since
-          // the alts are non-essential.
-          users,
-          dailyScheduleWeeks,
-          basisScheduleWeeks,
-        }));
-    });
-}
-
-/**
- * Wrapper around getScheduleData that is cheap to call. In most cases it
- * returns a cached version. The cache is stored for 30 minutes.
- * @returns {Promise} { users, dailyScheduleWeeks, basisScheduleWeeks }
-*/
-function getScheduleDataCacheWrapper() {
-  if (meetingpointData == null || new Date() - lastUpdate > 30 * 60 * 1000) { // 30 minutes
-    return getScheduleData().then((meetingpointData_) => {
-      lastUpdate = new Date();
-      meetingpointData = meetingpointData_;
-
-      return meetingpointData;
-    });
-  }
-
-  return Promise.resolve(meetingpointData);
-}
-
-// Debounce getScheduleDataCacheWrapper. This ensures that no requests will be
-// waited if a user requests the schedule data while the schedule data is
-// already being requested by another user.
-module.exports = debounce(getScheduleDataCacheWrapper);
diff --git a/src/server/routes/getSchedule.js b/src/server/routes/getSchedule.js
index 2ade31a..fb78e97 100644
--- a/src/server/routes/getSchedule.js
+++ b/src/server/routes/getSchedule.js
@@ -22,10 +22,10 @@ const express = require('express');
 
 const router = express.Router();
 
-const getScheduleData = require('../lib/schools/hetmml/getScheduleData');
-const getURLOfUser = require('../lib/schools/hetmml/getURLOfUser');
-const axios = require('../lib/schools/hetmml/axios');
-const parseSchedule = require('../lib/schools/hetmml/parseSchedule');
+const { getUsers } = require('../../shared/lib/db');
+const getURLOfUser = require('../../shared/lib/getURLOfUser');
+const axios = require('../../shared/lib/axios');
+const parseSchedule = require('../lib/parseSchedule');
 
 // copied from http://www.meetingpointmco.nl/Roosters-AL/doc/dagroosters/untisscripts.js,
 // were using the same code as they do to be sure that we always get the same
@@ -46,14 +46,14 @@ function currentWeekNumber() {
   return 1 + Math.ceil((firstThursday - target) / 604800000);
 }
 
-async function getSchedule(userType, userValue, week, scheduleType = 'dag') {
-  const { users } = await getScheduleData();
+async function getSchedule(userType, userName, week, scheduleType = 'dag') {
+  const users = await getUsers();
   const user = users.filter(user_ => (
-    user_.type === userType && user_.value === userValue
+    user_.type === userType && user_.name === userName
   ))[0];
 
   if (!user) {
-    throw new Error(`${userType}/${userValue} is not in the user index.`);
+    throw new Error(`${userType}/${userName} is not in the user index.`);
   }
 
   if (!week) {
diff --git a/src/server/routes/index.js b/src/server/routes/index.js
index d647210..b0e86ee 100644
--- a/src/server/routes/index.js
+++ b/src/server/routes/index.js
@@ -21,31 +21,13 @@
 const express = require('express');
 
 const router = express.Router();
-const getScheduleData = require('../lib/schools/hetmml/getScheduleData');
+const { getUsers } = require('../../shared/lib/db');
 
 /* GET home page. */
-router.get(['/', '/s/*', '/t/*', '/r/*', '/c/*'], (req, res) => {
-  getScheduleData().then(({ users, dailyScheduleWeeks, basisScheduleWeeks }) => {
-    const isBeta = process.env.BETA === '1';
-
-    const flags = [];
-    if (isBeta) {
-      flags.push('BETA');
-      flags.push('NO_FEATURE_DETECT');
-    } else if (req.query.nfd != null) {
-      flags.push('NO_FEATURE_DETECT');
-    }
-
-    const flagsStr = `var FLAGS = ${JSON.stringify(flags)};`;
-    const usersStr = `var USERS = ${JSON.stringify(users)};`;
-    const dailyScheduleWeeksStr = `var DAGROOSTER_WEEKS = ${JSON.stringify(dailyScheduleWeeks)}`;
-    const basisScheduleWeeksStr = `var BASISROOSTER_WEEKS = ${JSON.stringify(basisScheduleWeeks)}`;
-
+router.get(['/', '/student/*', '/teacher/*', '/room/*', '/class/*'], (req, res) => {
+  getUsers().then((users) => {
     res.render('index', {
-      flagsStr,
-      usersStr,
-      dailyScheduleWeeksStr,
-      basisScheduleWeeksStr,
+      usersStr: JSON.stringify(users),
     });
   });
   // .catch(() => {
diff --git a/src/server/lib/schools/hetmml/axios.js b/src/shared/lib/axios.js
index 2de6047..2de6047 100644
--- a/src/server/lib/schools/hetmml/axios.js
+++ b/src/shared/lib/axios.js
diff --git a/src/shared/lib/db.js b/src/shared/lib/db.js
new file mode 100644
index 0000000..cb0be09
--- /dev/null
+++ b/src/shared/lib/db.js
@@ -0,0 +1,62 @@
+/**
+ * Copyright (C) 2018 Noah Loomans
+ *
+ * This file is part of rooster.hetmml.nl.
+ *
+ * rooster.hetmml.nl is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * rooster.hetmml.nl is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with rooster.hetmml.nl.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const { Pool } = require('pg');
+
+const pool = new Pool({
+  database: process.env.PGDATABASE || 'rooster_hetmml_nl',
+});
+
+// the pool with emit an error on behalf of any idle clients
+// it contains if a backend error or network partition happens
+pool.on('error', (error) => {
+  console.error('Unexpected error on idle client', error);
+  // TODO: Do we want to exit here?
+  process.exit(-1);
+});
+
+async function insertUsers(users) {
+  await pool.query('TRUNCATE TABLE schedule_user');
+  const promises = users.map(user => (
+    pool.query(
+      'INSERT INTO schedule_user(key, type, name, alt_name, index) VALUES ($1, $2, $3, $4, $5)',
+      [user.key, user.type, user.name, user.altName, user.index],
+    )
+  ));
+  await Promise.all(promises);
+}
+
+async function getUsers() {
+  const { rows } = await pool.query(
+    'SELECT key, type, name, alt_name, index FROM schedule_user',
+  );
+
+  const users = rows.map(row => ({
+    key: row.key,
+    type: row.type,
+    name: row.name,
+    altName: row.altName,
+    index: row.index,
+  }));
+
+  return users;
+}
+
+module.exports = { insertUsers, getUsers };
diff --git a/src/server/lib/schools/hetmml/getURLOfUser.js b/src/shared/lib/getURLOfUser.js
index 5010207..934f235 100644
--- a/src/server/lib/schools/hetmml/getURLOfUser.js
+++ b/src/shared/lib/getURLOfUser.js
@@ -20,7 +20,15 @@
 
 const leftPad = require('left-pad'); // I imported this just to piss you off ;)
 
-function getURLOfUser(scheduleType, type, index, week) {
+const shortTypeOf = {
+  class: 'c',
+  teacher: 't',
+  student: 's',
+  room: 'r',
+};
+
+function getURLOfUser(scheduleType, longType, index, week) {
+  const type = shortTypeOf[longType];
   return `/${scheduleType}roosters/${leftPad(week, 2, '0')}/${type}/${type}${leftPad(index + 1, 5, '0')}.htm`;
 }
 
diff --git a/src/sync/index.js b/src/sync/index.js
new file mode 100644
index 0000000..3ccbad9
--- /dev/null
+++ b/src/sync/index.js
@@ -0,0 +1,97 @@
+/**
+ * Copyright (C) 2018 Noah Loomans
+ *
+ * This file is part of rooster.hetmml.nl.
+ *
+ * rooster.hetmml.nl is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * rooster.hetmml.nl is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with rooster.hetmml.nl.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const moment = require('moment');
+
+const axios = require('../shared/lib/axios');
+const { insertUsers } = require('../shared/lib/db');
+const getUrlOfUser = require('../shared/lib/getURLOfUser');
+const { scrapeUsers, scrapeAltName } = require('./scrapeScheduleData');
+
+moment.locale('nl');
+
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+async function fetchUsers() {
+  const navBar = await axios.get('/dagroosters/frames/navbar.htm');
+  const users = scrapeUsers(navBar.data);
+
+  return users;
+}
+
+async function fetchAltNameOfUser(user) {
+  const url = getUrlOfUser('dag', user.type, user.index, moment().week());
+  const schedule = await axios.get(url);
+  const altName = scrapeAltName(schedule.data);
+  return altName;
+}
+
+function simplifyNameOfUser(user) {
+  if (user.type === 'class') {
+    return {
+      ...user,
+      name: user.name.slice(2),
+    };
+  }
+
+  return user;
+}
+
+async function sync() {
+  const users = (await fetchUsers()).map(simplifyNameOfUser);
+
+  const alts = [];
+
+  /* eslint-disable no-restricted-syntax, no-await-in-loop */
+  const teachers = users.filter(user => user.type === 'teacher')
+    .slice(0, 3); // TODO: Remove this.
+  for (const teacher of teachers) {
+    await delay(500);
+    const altName = await fetchAltNameOfUser(teacher);
+    console.log(`${teacher.key}: ${altName}`);
+    if (altName) {
+      alts.push({ key: teacher.key, altName });
+    }
+  }
+  /* eslint-enable no-restricted-syntax, no-await-in-loop */
+
+  const usersWithAlts = users.map((user) => {
+    const { altName } = (
+      alts.find(altUser => altUser.key === user.key) || { altName: undefined }
+    );
+
+    return {
+      ...user,
+      altName,
+    };
+  });
+
+  await insertUsers(usersWithAlts);
+  console.log(usersWithAlts.filter(user => user.type === 'teacher'));
+}
+
+sync()
+  .then(() => {
+    process.exit();
+  })
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });
diff --git a/src/sync/scrapeScheduleData.js b/src/sync/scrapeScheduleData.js
new file mode 100644
index 0000000..fc7193a
--- /dev/null
+++ b/src/sync/scrapeScheduleData.js
@@ -0,0 +1,89 @@
+/**
+ * Copyright (C) 2018 Noah Loomans
+ *
+ * This file is part of rooster.hetmml.nl.
+ *
+ * rooster.hetmml.nl is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * rooster.hetmml.nl is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with rooster.hetmml.nl.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const cheerio = require('cheerio');
+const _ = require('lodash');
+
+/**
+ * Scrape all the valid users from a meetingpoint navbar.
+ * @param {string} html The html of a meetingpoint navbar.
+ * @returns {*}
+ * [
+ *   { type: 't', value: 'akh', index: 0 },
+ *   ...
+ *   { type: 's', value: '18561', index: 245 },
+ *   ...
+ *   { type: 'r', value: '008-mk', index: 2 },
+ *   ...
+ *   { type: 'c', value: '6-5H2', index: 23 },
+ *   ...
+ * ]
+ */
+function scrapeUsers(html) {
+  const page = cheerio.load(html);
+  const script = page('script').eq(1).text();
+
+  const regexs = [/var classes = \[(.+)\];/, /var teachers = \[(.+)\];/, /var rooms = \[(.+)\];/, /var students = \[(.+)\];/];
+  const items = regexs.map(regex => script.match(regex)[1].split(',').map(item => item.replace(/"/g, '')));
+
+  const classes = items[0].map((name, index) => ({
+    key: `c/${name}`,
+    type: 'class',
+    name,
+    index,
+  }));
+
+  const teachers = items[1].map((name, index) => ({
+    key: `t/${name}`,
+    type: 'teacher',
+    name,
+    index,
+  }));
+
+  const rooms = items[2].map((name, index) => ({
+    key: `r/${name}`,
+    type: 'room',
+    name,
+    index,
+  }));
+
+  const students = items[3].map((name, index) => ({
+    key: `s/${name}`,
+    type: 'student',
+    name,
+    index,
+  }));
+
+  return _.flatten([classes, teachers, rooms, students]);
+}
+
+/**
+ * scrape the alt text (the text next to the short code) from a
+ * specific meetingpoint schedule.
+ * @param {string} html The html of a specific meetingpoint schedule.
+ * @returns {string}
+ */
+function scrapeAltName(html) {
+  const page = cheerio.load(html);
+  return page('center > font').eq(2).text().trim() || undefined;
+}
+
+module.exports.scrapeUsers = scrapeUsers;
+module.exports.scrapeAltName = scrapeAltName;
author	Noah Loomans <noahloomans@gmail.com>	2018-07-07 17:34:02 +0200
committer	Noah Loomans <noahloomans@gmail.com>	2018-07-07 17:34:02 +0200
commit	5c265c04ad513d845a41c7866c3ed231c8d5e68e (patch)
tree	9a86b66958e59e4a07d0c8e10731667b05990816 /src
parent	8b1f29f2802a081a67151e47b9c52803d7cc568a (diff)