From 2481fcc745ccfea55054df09d7f17617653bc798 Mon Sep 17 00:00:00 2001 From: Glavin Wiechert Date: Sat, 6 May 2017 20:00:42 -0300 Subject: [PATCH 1/7] See #21. Add Dockerfile and docker-compose.yml for Resin.io + Raspberry Pi --- Dockerfile | 36 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 11 +++++++++++ 2 files changed, 47 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f38250c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +FROM resin/raspberry-pi-python:latest +MAINTAINER Glavin Wiechert + +RUN apt-get update + +# Install Node.js +ENV NODE_VERSION 7.8.0 +RUN curl -SLO "http://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-armv6l.tar.gz" \ + && echo "17244fea0ab0e94fbdae10b5998951c1d83fdab9d5b91209debdedc94a4fc7a1 node-v7.8.0-linux-armv6l.tar.gz" | sha256sum -c - \ + && tar -xzf "node-v$NODE_VERSION-linux-armv6l.tar.gz" -C /usr/local --strip-components=1 \ + && rm "node-v$NODE_VERSION-linux-armv6l.tar.gz" \ + && npm config set unsafe-perm true -g --unsafe-perm \ + && rm -rf /tmp/* + +# Install more dependencies +RUN apt-get install -y \ + python-numpy python-scipy + +# Install PostgreSQL +RUN apt-get install -y postgresql + +# Defines our working directory in container +WORKDIR /usr/src/app + +# Copy the application project +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY package.json . +RUN npm install + +# Enable systemd init system in container +# ENV INITSYSTEM on + +# Run on device +CMD "npm start" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8a81855 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2' + +services: + + issue-bot: + container_name: issue-bot + build: . + ports: + - "8080:80" + volumes: + - /data From 8ee2d00616832c9fd3ffe42fe7ad3d861ff7312a Mon Sep 17 00:00:00 2001 From: Glavin Wiechert Date: Sat, 6 May 2017 22:26:32 -0300 Subject: [PATCH 2/7] See #21. Fix CMD in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f38250c..53993b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,4 +33,4 @@ RUN npm install # ENV INITSYSTEM on # Run on device -CMD "npm start" \ No newline at end of file +CMD npm start \ No newline at end of file From 6c1cb3fd60ff7efb2a7d2fb70ec5c7f3f097da39 Mon Sep 17 00:00:00 2001 From: Glavin Wiechert Date: Mon, 8 May 2017 01:48:18 -0300 Subject: [PATCH 3/7] See #21. Setup PostgreSQL, create database and user, and more --- .dockerignore | 4 ++++ Dockerfile | 19 +++++++++++++++++-- config/custom-environment-variables.json | 7 +++++++ config/resin.json | 16 ++++++++++++++++ public/src/index.html | 2 +- public/src/js/components/app.jsx | 2 +- public/src/js/components/welcome.jsx | 2 +- scripts/create-database.sh | 18 ++++++++++++++++++ scripts/start.sh | 11 +++++++++++ src/index.js | 2 +- 10 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 .dockerignore create mode 100644 config/custom-environment-variables.json create mode 100644 config/resin.json create mode 100644 scripts/create-database.sh create mode 100755 scripts/start.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5ace6b7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +config/local.json +public/dist/ +node_modules/ +data/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 53993b8..8c2b761 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,12 +25,27 @@ WORKDIR /usr/src/app # Copy the application project COPY requirements.txt . RUN pip install -r requirements.txt - COPY package.json . RUN npm install +# Create PostgreSQL database +COPY scripts/create-database.sh scripts/create-database.sh +RUN chmod +x scripts/create-database.sh +RUN scripts/create-database.sh + +# Build +COPY webpack.config.babel.js \ + .babelrc \ + ./ +COPY scripts/ scripts/ +RUN chmod +x scripts/start.sh +COPY config/ config/ +COPY src/ src/ +COPY public/src/ public/src/ +RUN npm run build + # Enable systemd init system in container # ENV INITSYSTEM on # Run on device -CMD npm start \ No newline at end of file +CMD scripts/start.sh \ No newline at end of file diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json new file mode 100644 index 0000000..279ba1a --- /dev/null +++ b/config/custom-environment-variables.json @@ -0,0 +1,7 @@ +{ + "github": { + "token": "GITHUB_TOKEN", + "client_id": "GITHUB_CLIENT_ID", + "client_secret": "GITHUB_CLIENT_SECRET" + } +} diff --git a/config/resin.json b/config/resin.json new file mode 100644 index 0000000..9a91b98 --- /dev/null +++ b/config/resin.json @@ -0,0 +1,16 @@ +{ + "server": { + "base_url": "http://localhost:8080", + "port": 80 + }, + "app": { + "base_url": "http://localhost:8080" + }, + "db": { + "database": "issuemanager", + "username": "resin", + "password": "resin", + "host": "localhost", + "dialect": "postgres" + } +} diff --git a/public/src/index.html b/public/src/index.html index 1e2ca21..fcfef58 100644 --- a/public/src/index.html +++ b/public/src/index.html @@ -12,7 +12,7 @@
- + diff --git a/public/src/js/components/app.jsx b/public/src/js/components/app.jsx index dca9392..f045148 100644 --- a/public/src/js/components/app.jsx +++ b/public/src/js/components/app.jsx @@ -10,7 +10,7 @@ export default class App extends Component { super(props); this.state = { - socket: io('http://issue-manager.ngrok.io') + socket: io() }; } diff --git a/public/src/js/components/welcome.jsx b/public/src/js/components/welcome.jsx index e62a340..ad5e3fb 100644 --- a/public/src/js/components/welcome.jsx +++ b/public/src/js/components/welcome.jsx @@ -6,7 +6,7 @@ export default class Welcome extends Component {

Issue Manager!

Your automated Issue organizer and assistant.

-

Login with GitHub

+

Login with GitHub

); } diff --git a/scripts/create-database.sh b/scripts/create-database.sh new file mode 100644 index 0000000..83fe3ef --- /dev/null +++ b/scripts/create-database.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e + +# Start PostgreSQL +echo "Start PostgreSQL" +/etc/init.d/postgresql start + +# Create database +echo "Create PostgreSQL database" +echo "createdb issuemanager" | sudo -i -u postgres + +# Create User role +echo "Create PostgreSQL User" +echo "psql issuemanager --command=\"create user resin password 'resin';\"" | sudo -i -u postgres + +# Stop PostgreSQL +echo "Stop PostgreSQL" +/etc/init.d/postgresql stop diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 0000000..ade2aa1 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e + +# Start PostgreSQL +echo "Start PostgreSQL" +/etc/init.d/postgresql start + +# Start program +echo "Start IssueBot" +NODE_ENV=resin npm run start:server +# sleep 365d diff --git a/src/index.js b/src/index.js index b338765..85b7024 100644 --- a/src/index.js +++ b/src/index.js @@ -58,7 +58,7 @@ sequelize.sync({ }); githubOAuth.on('token', function(token, res) { - // console.log('here is your shiny new github oauth token', token); + console.log('here is your shiny new github oauth token', token); let access_token = token.access_token; // res.end(JSON.stringify(token)); res.redirect(`${config.get('app.base_url')}/#/setup?access_token=${access_token}`); From 4e5609caa7577ab5d7aab3b80cde374689393f21 Mon Sep 17 00:00:00 2001 From: Glavin Wiechert Date: Fri, 12 May 2017 23:28:12 -0300 Subject: [PATCH 4/7] See #21. Update Dockerfile to use glavin001/alpine-python2-numpy-scipy image --- Dockerfile | 30 ++++-------------------- config/custom-environment-variables.json | 9 ++++++- docker-compose.yml | 22 +++++++++++++++++ 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c2b761..b3b6847 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,22 @@ -FROM resin/raspberry-pi-python:latest +FROM glavin001/alpine-python2-numpy-scipy MAINTAINER Glavin Wiechert -RUN apt-get update +RUN apk update # Install Node.js -ENV NODE_VERSION 7.8.0 -RUN curl -SLO "http://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-armv6l.tar.gz" \ - && echo "17244fea0ab0e94fbdae10b5998951c1d83fdab9d5b91209debdedc94a4fc7a1 node-v7.8.0-linux-armv6l.tar.gz" | sha256sum -c - \ - && tar -xzf "node-v$NODE_VERSION-linux-armv6l.tar.gz" -C /usr/local --strip-components=1 \ - && rm "node-v$NODE_VERSION-linux-armv6l.tar.gz" \ - && npm config set unsafe-perm true -g --unsafe-perm \ - && rm -rf /tmp/* - -# Install more dependencies -RUN apt-get install -y \ - python-numpy python-scipy - -# Install PostgreSQL -RUN apt-get install -y postgresql +RUN apk add nodejs # Defines our working directory in container WORKDIR /usr/src/app # Copy the application project -COPY requirements.txt . -RUN pip install -r requirements.txt COPY package.json . RUN npm install -# Create PostgreSQL database -COPY scripts/create-database.sh scripts/create-database.sh -RUN chmod +x scripts/create-database.sh -RUN scripts/create-database.sh - # Build COPY webpack.config.babel.js \ .babelrc \ ./ -COPY scripts/ scripts/ -RUN chmod +x scripts/start.sh COPY config/ config/ COPY src/ src/ COPY public/src/ public/src/ @@ -48,4 +26,4 @@ RUN npm run build # ENV INITSYSTEM on # Run on device -CMD scripts/start.sh \ No newline at end of file +CMD npm run start:server diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 279ba1a..c8d351c 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -3,5 +3,12 @@ "token": "GITHUB_TOKEN", "client_id": "GITHUB_CLIENT_ID", "client_secret": "GITHUB_CLIENT_SECRET" + }, + "db": { + "database": "DB_DATABASE", + "username": "DB_USERNAME", + "password": "DB_PASSWORD", + "host": "DB_HOST", + "dialect": "DB_DIALECT" } -} +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8a81855..2c7a0e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,3 +9,25 @@ services: - "8080:80" volumes: - /data + environment: + - NODE_ENV=docker + - GITHUB_TOKEN=e7e0a931acf36746269b229424ba769f409a8d16 + - GITHUB_CLIENT_ID=33beb04a03e6bdee1049 + - GITHUB_CLIENT_SECRET=a9d801431462dd5c7aef4c62548975da4d3fcac5 + - DB_DATABASE=issuebot + - DB_USERNAME=issuebot + - DB_PASSWORD=issuebot + - DB_HOST=database + - DB_DIALECT=postgres + + database: + container_name: issue-bot-database + image: postgres:alpine + ports: + - "5432:5432" + volumes: + - /var/lib/postgresql/data + environment: + - POSTGRES_DB=issuebot + - POSTGRES_USER=issuebot + - POSTGRES_PASSWORD=issuebot \ No newline at end of file From b35f4dc8bbe526227772acb20b06ce7687080c8c Mon Sep 17 00:00:00 2001 From: Glavin Wiechert Date: Fri, 12 May 2017 23:29:28 -0300 Subject: [PATCH 5/7] See #21. Update numpy to 1.13.0rc1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1acff96..45f4ab4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ scikit-learn==0.17.1 -numpy==1.9.2 \ No newline at end of file +numpy==1.13.0rc1 \ No newline at end of file From bc09a29045d06fbc04d3f1541dad64707e0075fb Mon Sep 17 00:00:00 2001 From: Glavin Wiechert Date: Fri, 12 May 2017 23:29:51 -0300 Subject: [PATCH 6/7] See #21. Update dependencies and fix new bugs --- package.json | 41 +++++++++++++++--------------- public/src/index.html | 2 +- public/src/js/components/graph.jsx | 2 +- public/src/js/components/setup.jsx | 14 +++++----- scripts/create-database.sh | 2 +- scripts/start.sh | 11 -------- src/api/github.js | 2 +- src/api/repository.js | 12 +++++---- src/constants.js | 2 +- src/index.js | 2 ++ src/issues.js | 14 +++++----- webpack.config.babel.js | 23 ++++++++--------- 12 files changed, 60 insertions(+), 67 deletions(-) delete mode 100755 scripts/start.sh diff --git a/package.json b/package.json index c59d621..800a615 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "scripts:get-issues": "node scripts/get-issues.js", "scripts": "npm-run-all run:get-issues run:classify", "build": "webpack", + "build:watch": "webpack --watch", "start:webpack-watch": "webpack --watch", "start:webpack": "webpack-dev-server --hot --inline", "start:server": "node src", @@ -38,47 +39,47 @@ "bootstrap": "^3.3.6", "bootstrap-sass": "^3.3.6", "config": "^1.21.0", - "d3": "^3.5.17", + "d3": "3.5.17", "express": "^4.13.4", "font-awesome": "^4.6.3", - "font-awesome-webpack": "0.0.4", - "github": "^1.1.1", + "font-awesome-webpack": "0.0.5-beta.2", + "github": "^9.2.0", "github-oauth": "^0.2.2", - "github-webhook-handler": "^0.5.0", - "jquery": "^2.0.0", + "github-webhook-handler": "^0.6.0", + "jquery": "^3.2.1", "lodash": "^4.13.1", "mkdirp": "^0.5.1", - "parse-github-url": "^0.3.1", - "pg": "^5.1.0", + "parse-github-url": "^1.0.0", + "pg": "^6.1.5", "pg-hstore": "^2.3.2", "python-shell": "^0.4.0", "react": "^15.1.0", "react-dom": "^15.1.0", "react-if": "^2.0.5", - "react-router": "^2.4.1", + "react-router": "^3.0.5", "sequelize": "^3.23.3", - "socket.io": "^1.4.6" + "socket.io": "^2.0.1" }, "devDependencies": { "babel-core": "^6.9.1", - "babel-loader": "^6.2.4", + "babel-loader": "^7.0.0", "babel-plugin-transform-class-properties": "^6.9.1", "babel-preset-es2015": "^6.9.0", "babel-preset-react": "^6.5.0", "babel-preset-react-hmre": "^1.1.1", "chai": "^3.5.0", - "css-loader": "^0.23.1", - "file-loader": "^0.8.5", + "css-loader": "^0.28.1", + "file-loader": "^0.11.1", "less": "^2.7.1", - "less-loader": "^2.2.3", - "mocha": "^2.5.3", - "node-sass": "^3.7.0", - "npm-run-all": "^2.1.1", + "less-loader": "^4.0.3", + "mocha": "^3.3.0", + "node-sass": "^4.5.2", + "npm-run-all": "^4.0.2", "react-hot-loader": "^1.3.0", - "sass-loader": "^3.2.0", - "style-loader": "^0.13.1", + "sass-loader": "^6.0.5", + "style-loader": "^0.17.0", "url-loader": "^0.5.7", - "webpack": "^1.13.1", - "webpack-dev-server": "^1.14.1" + "webpack": "^2.5.1", + "webpack-dev-server": "^2.4.5" } } diff --git a/public/src/index.html b/public/src/index.html index fcfef58..4387b3c 100644 --- a/public/src/index.html +++ b/public/src/index.html @@ -13,7 +13,7 @@
- + diff --git a/public/src/js/components/graph.jsx b/public/src/js/components/graph.jsx index 70f50d9..9f46eb4 100644 --- a/public/src/js/components/graph.jsx +++ b/public/src/js/components/graph.jsx @@ -1,6 +1,6 @@ import React, { Component } from 'react'; import _ from 'lodash'; -import d3 from 'd3'; +import * as d3 from 'd3'; import ReactDOM from 'react-dom' // ***************************************************** diff --git a/public/src/js/components/setup.jsx b/public/src/js/components/setup.jsx index 867cf10..c9f341c 100644 --- a/public/src/js/components/setup.jsx +++ b/public/src/js/components/setup.jsx @@ -32,8 +32,9 @@ export default class Setup extends Component { componentDidMount() { console.log('Loading!'); - this.context.socket.emit(EVENTS.AUTHENTICATE, this.state.token, (err, user) => { - console.log(err, user); + this.context.socket.emit(EVENTS.AUTHENTICATE, this.state.token, (err, result) => { + console.log(err, result); + const user = result.data; this.setState({ user }); @@ -52,15 +53,14 @@ export default class Setup extends Component { isValidRepository: null, // Pending }); this.context.socket.emit(EVENTS.PARSE_REPOSITORY_URL, value, (repo) => { - // console.log('repo', repo); if (repo && repo.name) { this.context.socket.emit(EVENTS.GITHUB_REPO, { - user: repo.owner, + owner: repo.owner, repo: repo.name - }, (err, repo) => { + }, (err, result) => { this.setState({ - repo, - isValidRepository: !!repo + repo: result.data, + isValidRepository: Boolean(result && result.data) }); }); } else { diff --git a/scripts/create-database.sh b/scripts/create-database.sh index 83fe3ef..dcdc73e 100644 --- a/scripts/create-database.sh +++ b/scripts/create-database.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh set -e # Start PostgreSQL diff --git a/scripts/start.sh b/scripts/start.sh deleted file mode 100755 index ade2aa1..0000000 --- a/scripts/start.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Start PostgreSQL -echo "Start PostgreSQL" -/etc/init.d/postgresql start - -# Start program -echo "Start IssueBot" -NODE_ENV=resin npm run start:server -# sleep 365d diff --git a/src/api/github.js b/src/api/github.js index 3bb4774..530ded4 100644 --- a/src/api/github.js +++ b/src/api/github.js @@ -20,7 +20,7 @@ module.exports = function(socket) { */ socket.on(EVENTS.GITHUB_REPO, (options, cb) => { if (!socket.github) return cb(ERRORS.MISSING_GITHUB); - + console.log(options); socket.github.repos.get(options, function(err, res) { return cb(err && err.message, res); }); diff --git a/src/api/repository.js b/src/api/repository.js index 05db79f..1e6944d 100644 --- a/src/api/repository.js +++ b/src/api/repository.js @@ -43,6 +43,7 @@ module.exports = function(socket, io) { // Get all Issues for a GitHub Repository let {owner, name} = repo; + console.log("repo", repo); async.auto({ /** @@ -55,13 +56,13 @@ module.exports = function(socket, io) { }); socket.github.repos.get({ - user: owner, + owner, repo: name - }, (err, repo) => { + }, (err, { data }) => { if (err) { return cb(err); } - repo = Repository.transformFields(repo); + repo = Repository.transformFields(data); // Attach the authentication token for repository access later repo.token = socket.token; @@ -117,7 +118,8 @@ module.exports = function(socket, io) { let repo = results.repo; let {issues} = results; let repoId = repo.id; - Repository.upsert(results.repo) + console.log('repo', repo); + Repository.upsert(repo) .then(() => { progress({ task: REPOSITORY_SYNC_TASKS.DATABASE, @@ -168,7 +170,7 @@ module.exports = function(socket, io) { // let webhookUrl = `${config.get('server.base_url')}/webhook/${repo.owner}/${repo.name}`; let webhookUrl = `${config.get('server.base_url')}/webhook`; socket.github.repos.createHook({ - user: repo.owner, + owner: repo.owner, repo: repo.name, name: 'web', events: ['issues','issue_comment','pull_request'], diff --git a/src/constants.js b/src/constants.js index faf3058..f845dd6 100644 --- a/src/constants.js +++ b/src/constants.js @@ -6,7 +6,7 @@ module.exports = Object.freeze({ REPOSITORY_SYNC_PROGRESS: 'repository sync progress', PARSE_REPOSITORY_URL: 'repository validate url', GITHUB_USER: 'github /user', - GITHUB_REPO: 'github /repos/:user/:repo', + GITHUB_REPO: 'github /repos/:owner/:repo', GITHUB_USER_REPOS: 'github /user/repos', }, ERRORS: { diff --git a/src/index.js b/src/index.js index 85b7024..19e5bda 100644 --- a/src/index.js +++ b/src/index.js @@ -11,6 +11,8 @@ const {sequelize, Repository, Issue} = require("./models"); const createHandler = require('github-webhook-handler'); const webhookHandler = createHandler({ path: '/webhook', secret: config.get('github.webhook_secret') }); +console.log(config); + sequelize.sync({ // force: true }).then(() => { diff --git a/src/issues.js b/src/issues.js index a50c98c..473102f 100644 --- a/src/issues.js +++ b/src/issues.js @@ -15,20 +15,20 @@ const githubPageSize = 100; module.exports = { - getIssues(github, user, repo, progressCb) { + getIssues(github, owner, repo, progressCb) { return new Promise((resolve, reject) => { github.search.issues({ - q: `repo:${user}/${repo}`, + q: `repo:${owner}/${repo}`, page: 1, per_page: 1 - }, (err, results) => { + }, (err, { data }) => { if (err) { return reject(err); } - // console.log('search', err, results); - let numOfIssues = results.total_count; + console.log('search', err, data); + let numOfIssues = data.total_count; let completed = 0; const progress = () => { if (typeof progressCb === 'function') { @@ -45,12 +45,12 @@ module.exports = { let pages = range(1, parseInt(numOfIssues / githubPageSize) + 2, 1); async.map(pages, (page, cb) => { github.issues.getForRepo({ - user, + owner, repo, page, state: 'all', per_page: githubPageSize - }, (err, issues = []) => { + }, (err, { data: issues = [] } = {}) => { completed += issues.length; progress(); diff --git a/webpack.config.babel.js b/webpack.config.babel.js index ac54491..e55ad57 100644 --- a/webpack.config.babel.js +++ b/webpack.config.babel.js @@ -2,18 +2,17 @@ var webpack = require('webpack'); module.exports = { entry: { - // 'webpack-dev-server/client?http://localhost:3000', - // 'webpack/hot/only-dev-server', - // './public/src/js/app.jsx', - javascript: './public/src/js/app.jsx', - html: './public/src/index.html' + bundle: [ + './public/src/js/app.jsx', + './public/src/index.html' + ] }, output: { - path: './public/dist', - filename: 'app.bundle.js' + path: __dirname + '/public/dist', + filename: '[name].js' }, resolve: { - extensions: ["", ".webpack.js", ".web.js", ".js", ".jsx"] + extensions: [".webpack.js", ".web.js", ".js", ".jsx"] }, plugins: [ new webpack.HotModuleReplacementPlugin() @@ -22,11 +21,11 @@ module.exports = { loaders: [ { test: /\.html$/, - loader: "file?name=[name].[ext]" + loader: "file-loader?name=[name].[ext]" }, { test: /\.jsx?$/, exclude: /node_modules/, - loaders: ['react-hot', 'babel?'+JSON.stringify({ + loaders: ['react-hot-loader', 'babel-loader?'+JSON.stringify({ presets: [ 'react', 'es2015' ], @@ -37,10 +36,10 @@ module.exports = { })] }, { test: /\.css$/, - loader: 'style!css' + loader: 'style-loader!css-loader' }, { test: /\.scss$/, - loaders: ["style", "css", "sass"] + loaders: ["style-loader", "css-loader", "sass-loader"] }, // the url-loader uses DataUrls. // the file-loader emits files. From f3c66eca33b656d9af292bbb153b75afe4448682 Mon Sep 17 00:00:00 2001 From: Glavin Wiechert Date: Sun, 14 May 2017 01:22:42 -0300 Subject: [PATCH 7/7] See #21. Work on improving labels support --- .gitignore | 1 + docker-compose.yml | 6 +++--- package.json | 2 +- public/src/js/components/graph.jsx | 9 +++++++-- scripts/get-issues.js | 2 +- scripts/multi-label_classification.py | 4 ++-- scripts/train.js | 8 ++++---- src/api/index.js | 7 ++++++- src/api/repository.js | 2 +- src/classifier/classifier.py | 21 +++++++++++++-------- src/issues.js | 2 +- src/models/repository.js | 22 ++++++++++++++++------ 12 files changed, 56 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 679e3fe..a6ac1fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +keys/ data/ config/local.js config/local.json diff --git a/docker-compose.yml b/docker-compose.yml index 2c7a0e4..337fa95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,9 +11,9 @@ services: - /data environment: - NODE_ENV=docker - - GITHUB_TOKEN=e7e0a931acf36746269b229424ba769f409a8d16 - - GITHUB_CLIENT_ID=33beb04a03e6bdee1049 - - GITHUB_CLIENT_SECRET=a9d801431462dd5c7aef4c62548975da4d3fcac5 + - GITHUB_TOKEN + - GITHUB_CLIENT_ID + - GITHUB_CLIENT_SECRET - DB_DATABASE=issuebot - DB_USERNAME=issuebot - DB_PASSWORD=issuebot diff --git a/package.json b/package.json index 800a615..ac01eac 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "start:webpack-watch": "webpack --watch", "start:webpack": "webpack-dev-server --hot --inline", "start:server": "node src", - "start": "npm-run-all --parallel start:webpack start:server" + "start": "npm-run-all --parallel build:watch start:server" }, "repository": { "type": "git", diff --git a/public/src/js/components/graph.jsx b/public/src/js/components/graph.jsx index 9f46eb4..c7b67c2 100644 --- a/public/src/js/components/graph.jsx +++ b/public/src/js/components/graph.jsx @@ -99,7 +99,11 @@ export default class Graph extends Component { // } var updateNode = (selection) => { - selection.attr("transform", (d) => "translate(" + d.x + "," + d.y + ")"); + console.log('selection', selection); + selection.attr("transform", (d) => { + console.log('d', d); + return "translate(" + d.x + "," + d.y + ")" + }); }; var enterLink = (selection) => { @@ -142,7 +146,8 @@ export default class Graph extends Component { console.log('svg', svg, g); // .scaleExtent([1, 100]) svg.call(d3.behavior.zoom().on("zoom", () => { - g.attr("transform", "translate(" + d3.event.translate + ")" + " scale(" + d3.event.scale + ")") + // console.log('d3.event', d3.event); + d3.event && g.attr("transform", "translate(" + d3.event.translate + ")" + " scale(" + d3.event.scale + ")") })) .on("dblclick.zoom", null); diff --git a/scripts/get-issues.js b/scripts/get-issues.js index 5a433b2..ba70074 100644 --- a/scripts/get-issues.js +++ b/scripts/get-issues.js @@ -47,7 +47,7 @@ async.parallel([ ], (err, results) => { // Get contents for each Issue - // console.log(err, issues); + console.log(err, results); let issues = results[1]; issues = _.map(issues, (issue) => { issue.labels = _.map(issue.labels, 'name'); diff --git a/scripts/multi-label_classification.py b/scripts/multi-label_classification.py index 9227fbe..345acff 100644 --- a/scripts/multi-label_classification.py +++ b/scripts/multi-label_classification.py @@ -10,8 +10,8 @@ # Get JSON data user = "Glavin001" -repo = "test-issues" -# repo = "atom-beautify" +# repo = "test-issues" +repo = "atom-beautify" # repo = "atom-preview" # user = "reactjs" # repo = "redux" diff --git a/scripts/train.js b/scripts/train.js index 7f40d43..fc057c8 100644 --- a/scripts/train.js +++ b/scripts/train.js @@ -4,18 +4,18 @@ const { train } = require('../src/classifier'); const fs = require('fs'); // Get list of Issues for repository -let user = 'Glavin001'; +let owner = 'Glavin001'; let repo = 'atom-beautify'; // let user = 'reactjs'; // let repo = 'redux'; // let user = 'nodejs'; // let repo = 'node'; -let dataPath = path.resolve(__dirname, '../data/',user,repo); +let dataPath = path.resolve(__dirname, '../data/', owner, repo); const issues = require(path.resolve(dataPath, 'issues.json')); console.log('Issues: '+issues.length); - -train(issues) +const ignoreLabels = ['published', 'cannot-reproduce', 'high priority', 'in-progress', 'invalid', 'quick-todo', 'update-dependency', 'waiting-for-user-information', 'linux', 'mac']; +train('labels', [owner, repo, issues, ignoreLabels]) .then((results) => { console.log('finished!'); fs.writeFileSync(path.resolve(dataPath,'results.json'), JSON.stringify(results, undefined, 2)); diff --git a/src/api/index.js b/src/api/index.js index cac3330..08751d5 100644 --- a/src/api/index.js +++ b/src/api/index.js @@ -34,8 +34,13 @@ module.exports = function(socket, io) { const github = new GitHubApi({ debug: false, }); + // Create an authenticated GitHub client - github.authenticate({type: "oauth", token: token}); + github.authenticate({ + type: "oauth", + token: token + }); + // Test the client by attempting to get the user's information github.users.get({}, function(err, res) { if (err) { diff --git a/src/api/repository.js b/src/api/repository.js index 1e6944d..00c0ba8 100644 --- a/src/api/repository.js +++ b/src/api/repository.js @@ -220,7 +220,7 @@ module.exports = function(socket, io) { let {repo} = results; let {owner, name} = repo; - let ignoreLabels = []; // TODO + let ignoreLabels = ['published', 'cannot-reproduce', 'high priority', 'in-progress', 'invalid', 'quick-todo', 'update-dependency', 'waiting-for-user-information', 'linux', 'mac']; // TODO train('labels', [owner, name, issues, ignoreLabels]) .then((resp) => { diff --git a/src/classifier/classifier.py b/src/classifier/classifier.py index d322932..00bf2ef 100644 --- a/src/classifier/classifier.py +++ b/src/classifier/classifier.py @@ -108,7 +108,8 @@ def filter_list(items, ignore_items): return items def transform_issue(issue, ignore_labels=[]): - labels = filter_list(issue['labels'], ignore_labels) + # labels = filter_list(issue['labels'], ignore_labels) + labels = issue['labels'] milestone = issue['milestone'] if 'milestone' in issue else None title = (issue['title'] or "").encode('utf-8') body = (issue['body'] or "").encode('utf-8') @@ -139,7 +140,7 @@ def train_issues(owner, repo, issues, ignore_labels = []): else: label_counts[label] = 1 # Check for labels with insufficient number of examples - remove_labels = ['duplicate', 'wontfix'] + remove_labels = ['duplicate', 'wontfix'] + ignore_labels for label in label_counts.keys(): if label_counts[label] < k_folds: remove_labels.append(label) @@ -152,7 +153,8 @@ def train_issues(owner, repo, issues, ignore_labels = []): "error_message": ERRORS['InsufficientLabels'], "label_counts": label_counts, "all_labels": label_counts.keys(), - "remove_labels": remove_labels + "remove_labels": remove_labels, + "ignore_labels": ignore_labels }) number_train = [] @@ -166,10 +168,11 @@ def train_issues(owner, repo, issues, ignore_labels = []): (number, text, labels) = issue # Remove labels with insufficient number of examples - for label in remove_labels: - if label in labels: - labels.remove(label) - # print("removed label ",label) + labels = filter_list(labels, remove_labels) + # for label in remove_labels: + # if label in labels: + # labels.remove(label) + # # print("removed label ",label) if len(labels) > 0: # and issue['state'] == 'closed': # if milestone != None: @@ -192,7 +195,8 @@ def train_issues(owner, repo, issues, ignore_labels = []): "error_message": ERRORS['InsufficientLabels'], "label_counts": label_counts, "all_labels": label_counts.keys(), - "remove_labels": remove_labels + "remove_labels": remove_labels, + "ignore_labels": ignore_labels }) X_train = np.array(x_train) @@ -309,6 +313,7 @@ def train_issues(owner, repo, issues, ignore_labels = []): "label_counts": label_counts, "all_labels": label_counts.keys(), "remove_labels": remove_labels, + "ignore_labels": ignore_labels }, }) diff --git a/src/issues.js b/src/issues.js index 473102f..404bc03 100644 --- a/src/issues.js +++ b/src/issues.js @@ -23,7 +23,7 @@ module.exports = { q: `repo:${owner}/${repo}`, page: 1, per_page: 1 - }, (err, { data }) => { + }, (err, { data } = {}) => { if (err) { return reject(err); } diff --git a/src/models/repository.js b/src/models/repository.js index fdd0658..799c0ae 100644 --- a/src/models/repository.js +++ b/src/models/repository.js @@ -68,32 +68,42 @@ module.exports = function(sequelize, DataTypes) { debug: false, }); // Create an authenticated GitHub client - github.authenticate({type: "oauth", token: this.token}); + github.authenticate({ + type: "oauth", + token: this.token, + // key: config.get('github.client_id'), + // secret: config.get('github.client_secret'), + }); + return github; }, addLabelsToIssue(issue, labels) { + console.log('addLabelsToIssue', issue, labels); let {number} = issue; let {owner, name} = this; let github = this.getGitHubClient(); github.issues.addLabels({ - user: owner, + owner, repo: name, number, - body: labels - }); + labels + }) + .catch(err => console.error(err)); }, addCommentToIssue(issue, body) { + console.log('addCommentToIssue', issue, body); let {number} = issue; let {owner, name} = this; let github = this.getGitHubClient(); // Add signature to body body += config.get('comments.signature'); github.issues.createComment({ - user: owner, + owner, repo: name, number, body - }); + }) + .catch(err => console.error(err)); } }, classMethods: {