From 18275885f4da4b8d1da90d107eafd5002aa5b899 Mon Sep 17 00:00:00 2001 From: Xinyan He Date: Wed, 10 Aug 2022 23:45:19 -0400 Subject: [PATCH 1/7] Add D3 lib to Dockerfile --- Dockerfile.client | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.client b/Dockerfile.client index 279cd274..63df615b 100644 --- a/Dockerfile.client +++ b/Dockerfile.client @@ -5,6 +5,8 @@ COPY client/package.json /usr/src/app WORKDIR /usr/src/app RUN npm install --legacy-peer-deps +RUN npm install d3 --legacy-peer-deps + COPY client /usr/src/app RUN npm run build From 0359ee12c32ddf71e60b66f0fbd12b037489433e Mon Sep 17 00:00:00 2001 From: Xinyan He Date: Wed, 10 Aug 2022 23:48:31 -0400 Subject: [PATCH 2/7] Add the rank graph for fork activeness --- app/analyse/analyser.py | 20 ++- app/analyse/project_updater.py | 13 +- app/api/ForkList.py | 2 + client/src/App.js | 11 ++ client/src/FollowedRepositoryCard.jsx | 12 ++ client/src/ForkGraph.jsx | 97 +++++++++++++ client/src/ForkRank.jsx | 200 ++++++++++++++++++++++++++ 7 files changed, 346 insertions(+), 9 deletions(-) create mode 100644 client/src/ForkGraph.jsx create mode 100644 client/src/ForkRank.jsx diff --git a/app/analyse/analyser.py b/app/analyse/analyser.py index c66dc347..6e44db52 100644 --- a/app/analyse/analyser.py +++ b/app/analyse/analyser.py @@ -78,6 +78,20 @@ def get_active_forks(repo, access_token): return active_forks +def get_commit_number(repo, access_token): + + request_url = "https://api.github.com/repos/%s/stats/participation" % repo + + res = requests.get( + url=request_url, + headers={ + "Accept": "application/json", + "Authorization": "token {}".format(access_token), + }, + ) + repo_info = res.json() + return repo_info['all'] + @celery.task def start_analyse(repo, access_token): """Start analyse on repo using github_api_caller(contains personal access token) @@ -113,12 +127,12 @@ def start_analyse(repo, access_token): current_app.config["LOCAL_DATA_PATH"] + "/" + repo + "/forks_list.json" ) - active_forks = get_active_forks(repo,access_token ) + active_forks = get_active_forks(repo, access_token) if current_app.config["USE_LOCAL_FORKS_LIST"] and os.path.exists(forks_list_path): with open(forks_list_path) as read_file: repo_forks_list = json.load(read_file) - project_updater.start_update(repo, repo_info, repo_forks_list) + project_updater.start_update(repo, repo_info, repo_forks_list, access_token) return else: # repo_forks_list = github_api_caller.get("repos/%s/forks" % repo) @@ -135,7 +149,7 @@ def start_analyse(repo, access_token): print("finish fetch fork list for %s" % repo) - project_updater.start_update(repo, repo_info, active_forks) + project_updater.start_update(repo, repo_info, active_forks, access_token) # TODO: Fix email sending functionality # temporarily commented out to get working on local - laith diff --git a/app/analyse/project_updater.py b/app/analyse/project_updater.py index ca3e159c..caa40d4a 100644 --- a/app/analyse/project_updater.py +++ b/app/analyse/project_updater.py @@ -15,7 +15,7 @@ DATABASE_UPDATE_MODE = True class ForkUpdater: - def __init__(self, project_name, author, fork_info, code_clone_crawler): + def __init__(self, project_name, author, fork_info, code_clone_crawler, access_token): self.project_name = project_name self.author = author self.fork_name = fork_info["full_name"] @@ -24,6 +24,7 @@ def __init__(self, project_name, author, fork_info, code_clone_crawler): self.code_clone_crawler = code_clone_crawler self.diff_result_path = current_app.config['LOCAL_DATA_PATH'] + "/" + self.project_name + '/' + self.author + '/diff_result.json' self.all_tokens = [] + self.access_token = access_token # self.all_stemmed_tokens = [] self.all_lemmatize_tokens = [] @@ -87,9 +88,9 @@ def work(self): if os.path.exists(self.diff_result_path): with open(self.diff_result_path) as read_file: compare_result = json.load(read_file) - else: - # local file not exist - return + # else: + # # local file not exist + # return else: # If the compare result is not crawled, start to crawl. splitForkName = self.fork_name.split("/") @@ -175,7 +176,7 @@ def project_init(project_name, repo_info): analyser_progress="0%", ).save() -def start_update(project_name, repo_info, forks_info): +def start_update(project_name, repo_info, forks_info, access_token): Project.objects(project_name=project_name).update(activate_fork_number=get_activate_fork_number(forks_info)) forks_number = len(forks_info) forks_count = 0 @@ -183,7 +184,7 @@ def start_update(project_name, repo_info, forks_info): for fork in forks_info: forks_count += 1 try: - ForkUpdater(project_name, fork["owner"]["login"], fork, code_clone_crawler).work() + ForkUpdater(project_name, fork["owner"]["login"], fork, code_clone_crawler, access_token).work() except Exception as inst: print(inst) finally: diff --git a/app/api/ForkList.py b/app/api/ForkList.py index 120114ed..89937bf2 100644 --- a/app/api/ForkList.py +++ b/app/api/ForkList.py @@ -7,6 +7,7 @@ from ..models import User, ProjectFork, Project from ..analyse.compare_changes_crawler import fetch_commit_list, fetch_diff_code from ..analyse.analyser import get_active_forks +from ..analyse.analyser import get_commit_number from rake_nltk import Rake programming_languages = ["html", "js", "json", "py", "php", "css", "md", "babel", "yml", "java", "python", "javascript"] @@ -175,6 +176,7 @@ def get(self): "total_commit_number": fork["total_commit_number"], "last_committed_time": str(fork["last_committed_time"]), "created_time": str(fork["created_time"]), + "commit_freq": get_commit_number(fork["fork_name"], _user.github_access_token) } ) diff --git a/client/src/App.js b/client/src/App.js index c2f59ebf..727af0da 100644 --- a/client/src/App.js +++ b/client/src/App.js @@ -22,6 +22,7 @@ import LoginModal from "./LoginModal"; import ForkCluster from "./ForkCluster"; import { getUserLogin } from "./repository"; import Forklist from "./Forklist"; +import ForkGraph from "./ForkGraph"; import DrawerCard from "./DrawerCard"; const theme = createTheme({ @@ -163,6 +164,16 @@ const App = () => { ) } /> + + ) : ( + + ) + } + /> { + console.log("repo nav", repo); + navigate(`/visual/${repo}`, { replace: true }); + }; + return ( @@ -63,6 +68,13 @@ const FollowedRepositoryCard = ({ > View Forks + diff --git a/client/src/ForkGraph.jsx b/client/src/ForkGraph.jsx new file mode 100644 index 00000000..e3a91d0e --- /dev/null +++ b/client/src/ForkGraph.jsx @@ -0,0 +1,97 @@ + +import React, { useState, forwardRef, useEffect, useCallback } from "react"; +import { useParams } from "react-router-dom"; +import * as d3 from 'd3'; +import { getRepoForks } from "./repository"; +import Loading from "./common/Loading" +import ForkRank from "./ForkRank"; + +// const territories = ["External", "Far West", "Great Lakes"] + +// const data = [ +// {territory: "External", quarter: "Q1 2013", profit: 41119.6}, +// {territory: "External", quarter: "Q2 2013", profit: 36771.95}, +// {territory: "External", quarter: "Q3 2013", profit: 45251.75}, +// {territory: "External", quarter: "Q4 2013", profit: 17989.05}, +// {territory: "External", quarter: "Q1 2014", profit: 25182.9}, +// {territory: "External", quarter: "Q2 2014", profit: 54538.25}, +// {territory: "External", quarter: "Q3 2014", profit: 22339.65}, +// {territory: "External", quarter: "Q4 2014", profit: 26487.8}, +// {territory: "Far West", quarter: "Q1 2013", profit: 278130.95}, +// {territory: "Far West", quarter: "Q2 2013", profit: 355180.3}, +// {territory: "Far West", quarter: "Q3 2013", profit: 277655.1}, +// {territory: "Far West", quarter: "Q4 2013", profit: 339116.05}, +// {territory: "Far West", quarter: "Q1 2014", profit: 358637.15}, +// {territory: "Far West", quarter: "Q2 2014", profit: 378244.35}, +// {territory: "Far West", quarter: "Q3 2014", profit: 360947.75}, +// {territory: "Far West", quarter: "Q4 2014", profit: 313951.6}, +// {territory: "Great Lakes", quarter: "Q1 2013", profit: 280034.45}, +// {territory: "Great Lakes", quarter: "Q2 2013", profit: 319310.55}, +// {territory: "Great Lakes", quarter: "Q3 2013", profit: 332849.1}, +// {territory: "Great Lakes", quarter: "Q4 2013", profit: 270933.85}, +// {territory: "Great Lakes", quarter: "Q1 2014", profit: 302933.6}, +// {territory: "Great Lakes", quarter: "Q2 2014", profit: 378663.75}, +// {territory: "Great Lakes", quarter: "Q3 2014", profit: 308821.75}, +// {territory: "Great Lakes", quarter: "Q4 2014", profit: 343936.35}, +// ] + +// const quarters = ["Q1 2013", "Q2 2013", "Q3 2013", "Q4 2013", "Q1 2014", "Q2 2014", "Q3 2014", "Q4 2014"] + + +const ForkGraph = () => { + + const { repo1, repo2 } = useParams(); + const [fork, setFork] = useState(null); + const [data, setData] = useState(null); + const [fork_names, setForkNames] = useState(null); + + const getDataList = (forks_list) => { + //EXTRACT TOP TEN + let datalist = [] + for (let i = 0; i < forks_list.length && i < 10; i++) { + let fork = forks_list[i] + console.log(fork) + let commit_list = fork["commit_freq"] + for (let j = 0; j < commit_list.length; j++) { + let ith_week_dic = {} + ith_week_dic["quarter"] = j + ith_week_dic["profit"] = commit_list[j] + ith_week_dic["territory"] = fork["fork_name"] + datalist.push(ith_week_dic) + } + } + return datalist +} + +const getForkNames = (forks_list) => { + let fork_names = [] + for (let i = 0; i < forks_list.length && i < 10; i++) { + fork_names.push(forks_list[i]["fork_name"]) + } + return fork_names +} + +const fetchForks = useCallback(async (repo) => { + console.log('repo1',repo1); + console.log('repo2', repo2); + const response = await getRepoForks(repo); + console.log("Fetching forks list for ", repo) + console.log("forks list response", response.data.forks); + setFork(response.data.forks); + setData(getDataList(response.data.forks)) + setForkNames(getForkNames(response.data.forks)) +}, []); + + useEffect(() => { + const repo = repo1 + "/" + repo2; + fetchForks(repo); + }, [fetchForks]); + + return ( + // data && fork_names ? : + + data && fork_names ? : + ); +} + +export default ForkGraph; \ No newline at end of file diff --git a/client/src/ForkRank.jsx b/client/src/ForkRank.jsx new file mode 100644 index 00000000..fbf070cf --- /dev/null +++ b/client/src/ForkRank.jsx @@ -0,0 +1,200 @@ +import React from "react"; +import * as d3 from 'd3'; + +const ForkRank = ({ + data, + territories + }) => { + + const ref = React.useRef(); + + const quarters = Array.from(Array(52).keys()); + + React.useEffect(() => { + const height = 1000; + const width = 1800; + const padding = 25; + const margin = {left: 105, right: 105, top: 20, bottom: 50}; + + const svg = d3.select(ref.current); + // const svg = d3.select('.plot-area'); + + // draw dashed line + const seq = (start, length) => + Array.apply(null, {length: length}).map((d, i) => i + start); + + + const bx = d3.scalePoint() + .domain(seq(0, quarters.length)) + .range([0, width - margin.left - margin.right - padding * 2]) + + + //2. chart + const ti = new Map(territories.map((territory, i) => [territory, i])); + const qi = new Map(quarters.map((quarter, i) => [quarter, i])); + + const matrix = Array.from(ti, () => new Array(quarters.length).fill(null)); + for (const {territory, quarter, profit} of data) + matrix[ti.get(territory)][qi.get(quarter)] = {rank: 0, profit: +profit, next: null}; + + matrix.forEach((d) => { + for (let i = 0; i { + const array = []; + matrix.forEach((d) => array.push(d[i])); + array.sort((a, b) => b.profit - a.profit); + array.forEach((d, j) => d.rank = j); + }); + + //before step 2 + // get ranking + const chartData = matrix; + const len = quarters.length - 1; + const ranking = chartData.map((d, i) => ({territory: territories[i], first: d[0].rank, last: d[len].rank})); + // get color + const color = d3.scaleOrdinal(d3.schemeTableau10) + .domain(seq(0, ranking.length)) + + const left = ranking.sort((a, b) => a.first - b.first).map((d) => d.territory); + const right = ranking.sort((a, b) => a.last - b.last).map((d) => d.territory); + + const strokeWidth = d3.scaleOrdinal() + .domain(["default", "transit", "compact"]) + .range([5, bumpRadius * 2 + 2, 2]); + const drawingStyle = 'default'; + const bumpRadius = 13 + const by = d3.scalePoint() + .domain(seq(0, ranking.length)) + .range([margin.top, height - margin.bottom - padding]) + + function restore() { + series.transition().duration(500) + .attr("fill", s => color(s[0].rank)).attr("stroke", s => color(s[0].rank)); + restoreTicks(leftY); + restoreTicks(rightY); + + function restoreTicks(axis) { + axis.selectAll(".tick text") + .transition().duration(500) + .attr("font-weight", "normal").attr("fill", "black"); + } + } + + function highlight(e, d) { + this.parentNode.appendChild(this); + series.filter(s => s !== d) + .transition().duration(500) + .attr("fill", "#ddd").attr("stroke", "#ddd"); + markTick(leftY, 0); + markTick(rightY, quarters.length - 1); + + function markTick(axis, pos) { + axis.selectAll(".tick text").filter((s, i) => i === d[pos].rank) + .transition().duration(500) + .attr("font-weight", "bold") + .attr("fill", color(d[0].rank)); + } + } + + //dashed line + svg.append("g") + .attr("transform", `translate(${margin.left + padding},0)`) + .selectAll("path") + .data(seq(0, quarters.length)) + .join("path") + .attr("stroke", "#ccc") + .attr("stroke-width", 2) + .attr("stroke-dasharray", "5,5") + .attr("d", d => d3.line()([[bx(d), 0], [bx(d), height - margin.bottom]])); + + const series = svg.selectAll(".series") + .data(chartData) + .join("g") + .attr("class", "series") + .attr("opacity", 1) + .attr("fill", d => color(d[0].rank)) + .attr("stroke", d => color(d[0].rank)) + .attr("transform", `translate(${margin.left + padding},0)`) + .on("mouseover", highlight) + .on("mouseout", restore); + + + + + series.selectAll("path") + .data(d => d) + .join("path") + .attr("stroke-width", strokeWidth(drawingStyle)) + .attr("d", (d, i) => { + if (d.next) + return d3.line()([[bx(i), by(d.rank)], [bx(i + 1), by(d.next.rank)]]); + }) + + const title = g => g.append("title") + .text((d, i) => `${d.territory} - ${quarters[i]}\nRank: ${d.profit.rank + 1}\nProfit: ${d.profit.profit}`) + + const bumps = series.selectAll("g") + .data((d, i) => d.map(v => ({territory: territories[i], profit: v, first: d[0].rank}))) + .join("g") + .attr("transform", (d, i) => `translate(${bx(i)},${by(d.profit.rank)})`) + //.call(g => g.append("title").text((d, i) => `${d.territory} - ${quarters[i]}\n${toCurrency(d.profit.profit)}`)); + .call(title); + + const ax = d3.scalePoint() + .domain(quarters) + .range([margin.left + padding, width - margin.right - padding]); + + const y = d3.scalePoint() + .range([margin.top, height - margin.bottom - padding]); + + const compact = drawingStyle === "compact"; + bumps.append("circle").attr("r", compact ? 5 : bumpRadius); + bumps.append("text") + .attr("dy", compact ? "-0.75em" : "0.35em") + .attr("fill", compact ? null : "white") + .attr("stroke", "none") + .attr("text-anchor", "middle") + .style("font-weight", "bold") + .style("font-size", "14px") + .text(d => d.profit.rank + 1); + + + + const drawAxis = (g, x, y, axis, domain) => { + g.attr("transform", `translate(${x},${y})`) + .call(axis) + .selectAll(".tick text") + .attr("font-size", "12px"); + + if (!domain) g.select(".domain").remove(); + } + + + + //Axis + svg.append("g").call(g => drawAxis(g, 0, height - margin.top - margin.bottom + padding, d3.axisBottom(ax), true)); + const leftY = svg.append("g").call(g => drawAxis(g, margin.left, 0, d3.axisLeft(y.domain(left)))); + const rightY = svg.append("g").call(g => drawAxis(g, width - margin.right, 0, d3.axisRight(y.domain(right)))); + + }, + [data.length] + ); + + return ( + + + + ); +} +export default ForkRank; \ No newline at end of file From 9da94604c904c5aac98631b9b430a7f059193afc Mon Sep 17 00:00:00 2001 From: Xinyan He Date: Thu, 11 Aug 2022 11:28:05 -0400 Subject: [PATCH 3/7] Code Refactoring --- app/analyse/analyser.py | 8 +++---- app/analyse/project_updater.py | 5 ++-- client/src/ForkGraph.jsx | 43 +++------------------------------ client/src/ForkRank.jsx | 44 +++++++++++++++++----------------- 4 files changed, 31 insertions(+), 69 deletions(-) diff --git a/app/analyse/analyser.py b/app/analyse/analyser.py index 6e44db52..218cf860 100644 --- a/app/analyse/analyser.py +++ b/app/analyse/analyser.py @@ -89,8 +89,8 @@ def get_commit_number(repo, access_token): "Authorization": "token {}".format(access_token), }, ) - repo_info = res.json() - return repo_info['all'] + commit_info = res.json() + return commit_info['all'] @celery.task def start_analyse(repo, access_token): @@ -132,7 +132,7 @@ def start_analyse(repo, access_token): if current_app.config["USE_LOCAL_FORKS_LIST"] and os.path.exists(forks_list_path): with open(forks_list_path) as read_file: repo_forks_list = json.load(read_file) - project_updater.start_update(repo, repo_info, repo_forks_list, access_token) + project_updater.start_update(repo, repo_info, repo_forks_list) return else: # repo_forks_list = github_api_caller.get("repos/%s/forks" % repo) @@ -149,7 +149,7 @@ def start_analyse(repo, access_token): print("finish fetch fork list for %s" % repo) - project_updater.start_update(repo, repo_info, active_forks, access_token) + project_updater.start_update(repo, repo_info, active_forks) # TODO: Fix email sending functionality # temporarily commented out to get working on local - laith diff --git a/app/analyse/project_updater.py b/app/analyse/project_updater.py index caa40d4a..8f39631a 100644 --- a/app/analyse/project_updater.py +++ b/app/analyse/project_updater.py @@ -15,7 +15,7 @@ DATABASE_UPDATE_MODE = True class ForkUpdater: - def __init__(self, project_name, author, fork_info, code_clone_crawler, access_token): + def __init__(self, project_name, author, fork_info, code_clone_crawler): self.project_name = project_name self.author = author self.fork_name = fork_info["full_name"] @@ -24,7 +24,6 @@ def __init__(self, project_name, author, fork_info, code_clone_crawler, access_t self.code_clone_crawler = code_clone_crawler self.diff_result_path = current_app.config['LOCAL_DATA_PATH'] + "/" + self.project_name + '/' + self.author + '/diff_result.json' self.all_tokens = [] - self.access_token = access_token # self.all_stemmed_tokens = [] self.all_lemmatize_tokens = [] @@ -184,7 +183,7 @@ def start_update(project_name, repo_info, forks_info, access_token): for fork in forks_info: forks_count += 1 try: - ForkUpdater(project_name, fork["owner"]["login"], fork, code_clone_crawler, access_token).work() + ForkUpdater(project_name, fork["owner"]["login"], fork, code_clone_crawler).work() except Exception as inst: print(inst) finally: diff --git a/client/src/ForkGraph.jsx b/client/src/ForkGraph.jsx index e3a91d0e..e53b2f0f 100644 --- a/client/src/ForkGraph.jsx +++ b/client/src/ForkGraph.jsx @@ -6,42 +6,9 @@ import { getRepoForks } from "./repository"; import Loading from "./common/Loading" import ForkRank from "./ForkRank"; -// const territories = ["External", "Far West", "Great Lakes"] - -// const data = [ -// {territory: "External", quarter: "Q1 2013", profit: 41119.6}, -// {territory: "External", quarter: "Q2 2013", profit: 36771.95}, -// {territory: "External", quarter: "Q3 2013", profit: 45251.75}, -// {territory: "External", quarter: "Q4 2013", profit: 17989.05}, -// {territory: "External", quarter: "Q1 2014", profit: 25182.9}, -// {territory: "External", quarter: "Q2 2014", profit: 54538.25}, -// {territory: "External", quarter: "Q3 2014", profit: 22339.65}, -// {territory: "External", quarter: "Q4 2014", profit: 26487.8}, -// {territory: "Far West", quarter: "Q1 2013", profit: 278130.95}, -// {territory: "Far West", quarter: "Q2 2013", profit: 355180.3}, -// {territory: "Far West", quarter: "Q3 2013", profit: 277655.1}, -// {territory: "Far West", quarter: "Q4 2013", profit: 339116.05}, -// {territory: "Far West", quarter: "Q1 2014", profit: 358637.15}, -// {territory: "Far West", quarter: "Q2 2014", profit: 378244.35}, -// {territory: "Far West", quarter: "Q3 2014", profit: 360947.75}, -// {territory: "Far West", quarter: "Q4 2014", profit: 313951.6}, -// {territory: "Great Lakes", quarter: "Q1 2013", profit: 280034.45}, -// {territory: "Great Lakes", quarter: "Q2 2013", profit: 319310.55}, -// {territory: "Great Lakes", quarter: "Q3 2013", profit: 332849.1}, -// {territory: "Great Lakes", quarter: "Q4 2013", profit: 270933.85}, -// {territory: "Great Lakes", quarter: "Q1 2014", profit: 302933.6}, -// {territory: "Great Lakes", quarter: "Q2 2014", profit: 378663.75}, -// {territory: "Great Lakes", quarter: "Q3 2014", profit: 308821.75}, -// {territory: "Great Lakes", quarter: "Q4 2014", profit: 343936.35}, -// ] - -// const quarters = ["Q1 2013", "Q2 2013", "Q3 2013", "Q4 2013", "Q1 2014", "Q2 2014", "Q3 2014", "Q4 2014"] - - const ForkGraph = () => { const { repo1, repo2 } = useParams(); - const [fork, setFork] = useState(null); const [data, setData] = useState(null); const [fork_names, setForkNames] = useState(null); @@ -54,8 +21,8 @@ const ForkGraph = () => { let commit_list = fork["commit_freq"] for (let j = 0; j < commit_list.length; j++) { let ith_week_dic = {} - ith_week_dic["quarter"] = j - ith_week_dic["profit"] = commit_list[j] + ith_week_dic["week"] = j + ith_week_dic["commits"] = commit_list[j] ith_week_dic["territory"] = fork["fork_name"] datalist.push(ith_week_dic) } @@ -76,8 +43,6 @@ const fetchForks = useCallback(async (repo) => { console.log('repo2', repo2); const response = await getRepoForks(repo); console.log("Fetching forks list for ", repo) - console.log("forks list response", response.data.forks); - setFork(response.data.forks); setData(getDataList(response.data.forks)) setForkNames(getForkNames(response.data.forks)) }, []); @@ -88,9 +53,7 @@ const fetchForks = useCallback(async (repo) => { }, [fetchForks]); return ( - // data && fork_names ? : - - data && fork_names ? : + data && fork_names ? : ); } diff --git a/client/src/ForkRank.jsx b/client/src/ForkRank.jsx index fbf070cf..8803b63f 100644 --- a/client/src/ForkRank.jsx +++ b/client/src/ForkRank.jsx @@ -3,16 +3,16 @@ import * as d3 from 'd3'; const ForkRank = ({ data, - territories + fork_names }) => { const ref = React.useRef(); - const quarters = Array.from(Array(52).keys()); + const weeks = Array.from(Array(52).keys()); React.useEffect(() => { const height = 1000; - const width = 1800; + const width = 1500; const padding = 25; const margin = {left: 105, right: 105, top: 20, bottom: 50}; @@ -25,35 +25,35 @@ const ForkRank = ({ const bx = d3.scalePoint() - .domain(seq(0, quarters.length)) + .domain(seq(0, weeks.length)) .range([0, width - margin.left - margin.right - padding * 2]) //2. chart - const ti = new Map(territories.map((territory, i) => [territory, i])); - const qi = new Map(quarters.map((quarter, i) => [quarter, i])); + const ti = new Map(fork_names.map((territory, i) => [territory, i])); + const qi = new Map(weeks.map((week, i) => [week, i])); - const matrix = Array.from(ti, () => new Array(quarters.length).fill(null)); - for (const {territory, quarter, profit} of data) - matrix[ti.get(territory)][qi.get(quarter)] = {rank: 0, profit: +profit, next: null}; + const matrix = Array.from(ti, () => new Array(weeks.length).fill(null)); + for (const {territory, week, commits} of data) + matrix[ti.get(territory)][qi.get(week)] = {rank: 0, commits: +commits, next: null}; matrix.forEach((d) => { for (let i = 0; i { + weeks.forEach((d, i) => { const array = []; matrix.forEach((d) => array.push(d[i])); - array.sort((a, b) => b.profit - a.profit); + array.sort((a, b) => b.commits - a.commits); array.forEach((d, j) => d.rank = j); }); //before step 2 // get ranking const chartData = matrix; - const len = quarters.length - 1; - const ranking = chartData.map((d, i) => ({territory: territories[i], first: d[0].rank, last: d[len].rank})); + const len = weeks.length - 1; + const ranking = chartData.map((d, i) => ({territory: fork_names[i], first: d[0].rank, last: d[len].rank})); // get color const color = d3.scaleOrdinal(d3.schemeTableau10) .domain(seq(0, ranking.length)) @@ -89,7 +89,7 @@ const ForkRank = ({ .transition().duration(500) .attr("fill", "#ddd").attr("stroke", "#ddd"); markTick(leftY, 0); - markTick(rightY, quarters.length - 1); + markTick(rightY, weeks.length - 1); function markTick(axis, pos) { axis.selectAll(".tick text").filter((s, i) => i === d[pos].rank) @@ -103,7 +103,7 @@ const ForkRank = ({ svg.append("g") .attr("transform", `translate(${margin.left + padding},0)`) .selectAll("path") - .data(seq(0, quarters.length)) + .data(seq(0, weeks.length)) .join("path") .attr("stroke", "#ccc") .attr("stroke-width", 2) @@ -134,17 +134,17 @@ const ForkRank = ({ }) const title = g => g.append("title") - .text((d, i) => `${d.territory} - ${quarters[i]}\nRank: ${d.profit.rank + 1}\nProfit: ${d.profit.profit}`) + .text((d, i) => `${d.territory} - ${weeks[i]}\nRank: ${d.commits.rank + 1}\nProfit: ${d.commits.commits}`) const bumps = series.selectAll("g") - .data((d, i) => d.map(v => ({territory: territories[i], profit: v, first: d[0].rank}))) + .data((d, i) => d.map(v => ({territory: fork_names[i], commits: v, first: d[0].rank}))) .join("g") - .attr("transform", (d, i) => `translate(${bx(i)},${by(d.profit.rank)})`) - //.call(g => g.append("title").text((d, i) => `${d.territory} - ${quarters[i]}\n${toCurrency(d.profit.profit)}`)); + .attr("transform", (d, i) => `translate(${bx(i)},${by(d.commits.rank)})`) + //.call(g => g.append("title").text((d, i) => `${d.territory} - ${weeks[i]}\n${toCurrency(d.commits.commits)}`)); .call(title); const ax = d3.scalePoint() - .domain(quarters) + .domain(weeks) .range([margin.left + padding, width - margin.right - padding]); const y = d3.scalePoint() @@ -159,7 +159,7 @@ const ForkRank = ({ .attr("text-anchor", "middle") .style("font-weight", "bold") .style("font-size", "14px") - .text(d => d.profit.rank + 1); + .text(d => d.commits.rank + 1); @@ -188,7 +188,7 @@ const ForkRank = ({ ref={ref} style={{ height: 1000, - width: 2000, + width: 1500, marginRight: "0px", marginLeft: "0px", }} From a521ce4bc22643b41b6f93f95aa3adddf0ebe924 Mon Sep 17 00:00:00 2001 From: XinyanHe Date: Wed, 24 Aug 2022 16:28:13 -0400 Subject: [PATCH 4/7] Add progress bar --- Dockerfile.client | 2 +- app/__init__.py | 7 ++ app/analyse/analyser.py | 22 ++++- app/analyse/project_updater.py | 2 +- app/api/FollowRepository.py | 4 + app/api/ForkList.py | 82 +++++++-------- app/api/Progress.py | 71 +++++++++++++ client/src/FollowedRepositoryCard.jsx | 14 +-- client/src/ForkGraph.jsx | 137 +++++++++++++++++++++++--- client/src/ForkRank.jsx | 51 +++++----- client/src/Forklist.jsx | 32 +++++- client/src/ImportRepositoryCard.jsx | 6 +- client/src/SearchGithubRow.jsx | 9 +- client/src/common/Filter.jsx | 2 +- client/src/common/LinearLoading.jsx | 48 +++++++++ client/src/common/Loading.jsx | 2 +- client/src/repository/index.js | 23 +++++ 17 files changed, 414 insertions(+), 100 deletions(-) create mode 100644 app/api/Progress.py create mode 100644 client/src/common/LinearLoading.jsx diff --git a/Dockerfile.client b/Dockerfile.client index 63df615b..0dd169a9 100644 --- a/Dockerfile.client +++ b/Dockerfile.client @@ -6,7 +6,7 @@ COPY client/package.json /usr/src/app WORKDIR /usr/src/app RUN npm install --legacy-peer-deps RUN npm install d3 --legacy-peer-deps - +RUN npm install react-select --legacy-peer-deps COPY client /usr/src/app RUN npm run build diff --git a/app/__init__.py b/app/__init__.py index be1c8299..3784e7dd 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -13,6 +13,7 @@ from .api.FollowRepository import FollowRepository from .api.Auth import Auth from .api.ForkList import ForkList +from .api.Progress import Progress from .api.ForkClustering import ForkClustering from .db import initialize_db from .loginmanager import login_manager @@ -105,6 +106,12 @@ def serve(path): resource_class_kwargs={"jwt": jwt}, ) + api.add_resource( + Progress, + "/flask/progress", + resource_class_kwargs={"jwt": jwt}, + ) + # TODO: get correct host, broker and backend depending on environment redis_host = "redis://redis:6379/0" celery.conf.broker_url = redis_host diff --git a/app/analyse/analyser.py b/app/analyse/analyser.py index 218cf860..626a4400 100644 --- a/app/analyse/analyser.py +++ b/app/analyse/analyser.py @@ -78,7 +78,7 @@ def get_active_forks(repo, access_token): return active_forks -def get_commit_number(repo, access_token): +def get_commit_number_per_week(repo, access_token): request_url = "https://api.github.com/repos/%s/stats/participation" % repo @@ -90,7 +90,25 @@ def get_commit_number(repo, access_token): }, ) commit_info = res.json() - return commit_info['all'] + print(commit_info) + if ('all' in commit_info.keys()): + return commit_info['all'] + else: + return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + +def get_commit_number_per_hour(repo, access_token): + + request_url = "https://api.github.com/repos/%s/stats/commit_activity" % repo + + res = requests.get( + url=request_url, + headers={ + "Accept": "application/json", + "Authorization": "token {}".format(access_token), + }, + ) + commit_info = res.json() + return commit_info @celery.task def start_analyse(repo, access_token): diff --git a/app/analyse/project_updater.py b/app/analyse/project_updater.py index 8f39631a..a00f116a 100644 --- a/app/analyse/project_updater.py +++ b/app/analyse/project_updater.py @@ -175,7 +175,7 @@ def project_init(project_name, repo_info): analyser_progress="0%", ).save() -def start_update(project_name, repo_info, forks_info, access_token): +def start_update(project_name, repo_info, forks_info): Project.objects(project_name=project_name).update(activate_fork_number=get_activate_fork_number(forks_info)) forks_number = len(forks_info) forks_count = 0 diff --git a/app/api/FollowRepository.py b/app/api/FollowRepository.py index ed04e16e..e5e58b87 100644 --- a/app/api/FollowRepository.py +++ b/app/api/FollowRepository.py @@ -71,6 +71,9 @@ def post(self): else: add_repo(_user.username, repo, res) db_followed_project(_user, repo) + print("lalalal zzzzzz") + r = db_find_project(repo) + print(r["analyser_progress"]) msg = ( "The repo (%s) starts loading into INFOX. We will send you an email when it is finished. Please wait." % repo @@ -90,4 +93,5 @@ def post(self): "timesForked": res["forks_count"], "repo": res["full_name"], }, + "analyser_progress": db_find_project(repo)["analyser_progress"] } diff --git a/app/api/ForkList.py b/app/api/ForkList.py index 89937bf2..661c4a2f 100644 --- a/app/api/ForkList.py +++ b/app/api/ForkList.py @@ -7,7 +7,8 @@ from ..models import User, ProjectFork, Project from ..analyse.compare_changes_crawler import fetch_commit_list, fetch_diff_code from ..analyse.analyser import get_active_forks -from ..analyse.analyser import get_commit_number +from ..analyse.analyser import get_commit_number_per_week +from ..analyse.analyser import get_commit_number_per_hour from rake_nltk import Rake programming_languages = ["html", "js", "json", "py", "php", "css", "md", "babel", "yml", "java", "python", "javascript"] @@ -90,7 +91,7 @@ def get(self): request_url = "http://localhost:5000/flask/follow" if not forks_info: - + return {"forks": []} response = requests.post(request_url, json={"repo":repo}, headers={"Authorization": request.headers.get("Authorization")}) @@ -100,41 +101,43 @@ def get(self): # active_forks = get_active_forks(repoName, _user.github_access_token) active_forks = get_active_forks(repoName, _user.github_access_token) - alreadyAnalzyed = False - - for fork in forks_info: - if len(fork["key_words"]) > 0: - alreadyAnalzyed = True - break - if not alreadyAnalzyed: - for fork in active_forks: - fork_name = fork["full_name"][: -(len(fork["name"]) + 1)] - commit_msgs = fetch_commit_list(repoName, fork_name) - code_changes = fetch_diff_code(repoName, fork_name) - - sentences = [] - for msg in commit_msgs: - sentences.append(msg["title"]) - - for fork_info in code_changes: - if fork_info["file_full_name"]: - sentences.append(fork_info["file_full_name"]) - - # if fork_info["added_code"]: - # sentences.append(fork_info["added_code"]) - - if sentences: - rake.extract_keywords_from_sentences(sentences) - key_words[fork_name] = rake.get_ranked_phrases() - print("key words found for ", fork_name, ":", key_words[fork_name]) - ProjectFork.objects(fork_name=fork["full_name"]).update( - key_words=key_words[fork_name] - ) - else: - print("No keywords found for", fork_name) - ProjectFork.objects(fork_name=fork["full_name"]).update( - key_words=[] - ) + print("active fork") + print(len(active_forks)) +# alreadyAnalzyed = False + +# for fork in forks_info: +# if len(fork["key_words"]) > 0: +# alreadyAnalzyed = True +# break +# if not alreadyAnalzyed: +# for fork in active_forks: +# fork_name = fork["full_name"][: -(len(fork["name"]) + 1)] +# commit_msgs = fetch_commit_list(repoName, fork_name) +# code_changes = fetch_diff_code(repoName, fork_name) +# +# sentences = [] +# for msg in commit_msgs: +# sentences.append(msg["title"]) +# +# for fork_info in code_changes: +# if fork_info["file_full_name"]: +# sentences.append(fork_info["file_full_name"]) +# +# # if fork_info["added_code"]: +# # sentences.append(fork_info["added_code"]) +# +# if sentences: +# rake.extract_keywords_from_sentences(sentences) +# key_words[fork_name] = rake.get_ranked_phrases() +# print("key words found for ", fork_name, ":", key_words[fork_name]) +# ProjectFork.objects(fork_name=fork["full_name"]).update( +# key_words=key_words[fork_name] +# ) +# else: +# print("No keywords found for", fork_name) +# ProjectFork.objects(fork_name=fork["full_name"]).update( +# key_words=[] +# ) # for key, value in key_words.items(): # for word in value: # if not word.isnumeric(): @@ -149,7 +152,7 @@ def get(self): # print("Common words found by rake", top_common_words) - # forks_info = ProjectFork.objects(project_name=repoName) + forks_info = ProjectFork.objects(project_name=repoName) # db_keyword_dict = {} # top_common_words = {} # for fork in forks_info: @@ -176,7 +179,8 @@ def get(self): "total_commit_number": fork["total_commit_number"], "last_committed_time": str(fork["last_committed_time"]), "created_time": str(fork["created_time"]), - "commit_freq": get_commit_number(fork["fork_name"], _user.github_access_token) + "weekly_commit_freq": get_commit_number_per_week(fork["fork_name"], _user.github_access_token), + "hourly_commit_freq": get_commit_number_per_hour(fork["fork_name"], _user.github_access_token), } ) diff --git a/app/api/Progress.py b/app/api/Progress.py new file mode 100644 index 00000000..8f9a61dd --- /dev/null +++ b/app/api/Progress.py @@ -0,0 +1,71 @@ +from flask_restful import Resource +from flask_jwt_extended import get_jwt_identity +from flask_jwt_extended import jwt_required +import json +from flask import request +import requests +from ..models import User, ProjectFork, Project +from ..analyse.compare_changes_crawler import fetch_commit_list, fetch_diff_code +from ..analyse.analyser import get_active_forks +from ..analyse.analyser import get_commit_number_per_week +from ..analyse.analyser import get_commit_number_per_hour +from rake_nltk import Rake + +def db_find_project(project_name): + return Project.objects(project_name=project_name).first() + + +class Progress(Resource): + def __init__(self, jwt): + self.jwt = jwt + + @jwt_required() + def post(self): + + current_user = get_jwt_identity() + _user = User.objects(username=current_user).first() + + req_data = request.get_json() + repoName = req_data.get("repo") + index = req_data.get("index") + repo = repoName + + forks_info = ProjectFork.objects(project_name=repo) + fork = forks_info[index] + return_list = [] + + return_list.append( + { + "fork_name": fork["fork_name"], + "project_name": fork["project_name"], + "num_changed_files": fork["total_changed_file_number"], + "num_changed_lines": fork["total_changed_line_number"], + "changed_files": fork["file_list"], + "key_words": fork["key_words"], + "tags": fork["tags"], + "total_commit_number": fork["total_commit_number"], + "last_committed_time": str(fork["last_committed_time"]), + "created_time": str(fork["created_time"]), + "weekly_commit_freq": get_commit_number_per_week(fork["fork_name"], _user.github_access_token), + "hourly_commit_freq": get_commit_number_per_hour(fork["fork_name"], _user.github_access_token), + } + ) + + return {"forks": return_list} + + @jwt_required() + def get(self): + + + current_user = get_jwt_identity() + _user = User.objects(username=current_user).first() + + req_data = request.args + repoName = req_data.get("repo") + repo = repoName + + forks_info = ProjectFork.objects(project_name=repo) + + return len(forks_info) + + diff --git a/client/src/FollowedRepositoryCard.jsx b/client/src/FollowedRepositoryCard.jsx index 8461de9f..fe45019b 100644 --- a/client/src/FollowedRepositoryCard.jsx +++ b/client/src/FollowedRepositoryCard.jsx @@ -14,6 +14,7 @@ import RemoveButton from "./common/RemoveButton"; import { SECONDARY, TERTIARY } from "./common/constants"; import { deleteUserRepository } from "./repository"; import SignalCellularAltIcon from "@mui/icons-material/SignalCellularAlt"; +import Assessment from "@mui/icons-material/Assessment"; import { useNavigate } from "react-router"; import { PRIMARY, REMOVE } from "./common/constants"; @@ -54,12 +55,11 @@ const FollowedRepositoryCard = ({ style={{ background: SECONDARY }} expandIcon={} > - - + + {repo} - - + + + - - - diff --git a/client/src/ForkGraph.jsx b/client/src/ForkGraph.jsx index e53b2f0f..7fb03602 100644 --- a/client/src/ForkGraph.jsx +++ b/client/src/ForkGraph.jsx @@ -1,50 +1,146 @@ import React, { useState, forwardRef, useEffect, useCallback } from "react"; +import Select from 'react-select'; import { useParams } from "react-router-dom"; import * as d3 from 'd3'; import { getRepoForks } from "./repository"; -import Loading from "./common/Loading" +import { getActiveForksNum } from "./repository"; +import { postProgress } from "./repository"; +import LinearLoading from "./common/LinearLoading" import ForkRank from "./ForkRank"; const ForkGraph = () => { + const options = [ + { value: 'weekly', label: 'Weekly commits within last year' }, + { value: 'daily', label: 'Daily commits within last month' }, + ]; + const { repo1, repo2 } = useParams(); const [data, setData] = useState(null); - const [fork_names, setForkNames] = useState(null); + const [selection, setSelectionState] = useState("weekly"); + const [interval, setIntervalState] = useState(null); + const [forkNames, setForkNames] = useState(null); + const [forkList, setForkList] = useState(null); + const [activeForksNum, setActiveForksNum] = useState(0); + const [progress, setProgress] = useState(0); + const [counter, setCounter] = useState(0); const getDataList = (forks_list) => { //EXTRACT TOP TEN let datalist = [] - for (let i = 0; i < forks_list.length && i < 10; i++) { + for (let i = 0; i < forks_list.length; i++) { let fork = forks_list[i] console.log(fork) - let commit_list = fork["commit_freq"] + let commit_list = fork["weekly_commit_freq"] for (let j = 0; j < commit_list.length; j++) { let ith_week_dic = {} ith_week_dic["week"] = j ith_week_dic["commits"] = commit_list[j] - ith_week_dic["territory"] = fork["fork_name"] + ith_week_dic["fork_name"] = fork["fork_name"] datalist.push(ith_week_dic) } } return datalist } +const getDailyCommit = (forks_list) => { + //EXTRACT TOP TEN + let datalist = [] + for (let i = 0; i < forks_list.length; i++) { + let fork = forks_list[i] + let commit_list; + if (Object.keys(fork["hourly_commit_freq"]).length === 0){ + commit_list = [{"days": new Array(7).fill(0)}, + {"days": new Array(7).fill(0)}, + {"days": new Array(7).fill(0)}, + {"days": new Array(7).fill(0)}] + } else { + commit_list = fork["hourly_commit_freq"].slice(-4) + } + let index = 0; + for (let j = 0; j < 4; j++) { + for(let k = 0 ; k < 7; k++){ + let ith_week_dic = {} + ith_week_dic["week"] = index + ith_week_dic["commits"] = commit_list[j]["days"][k] + ith_week_dic["fork_name"] = fork["fork_name"] + index += 1 + datalist.push(ith_week_dic) + } + } + } + return datalist +} + +const updateData = (selection) => { + switch(selection) { + case 'weekly': + setIntervalState(Array.from(Array(52).keys())) + setData(getDataList(forkList)) + break; + case 'daily': + setIntervalState(Array.from(Array(28).keys())); + setData(getDailyCommit(forkList)); + break; + default: + // code block + } +} + +const handleChange = (selectedOption) => { + setSelectionState(selectedOption); + console.log(`Option selected:`, selectedOption); + updateData(selectedOption['value']) + console.log(data) + console.log(interval) +}; + const getForkNames = (forks_list) => { - let fork_names = [] - for (let i = 0; i < forks_list.length && i < 10; i++) { - fork_names.push(forks_list[i]["fork_name"]) + let forkNames = [] + // for (let i = 0; i < forks_list.length && i < 10; i++) { + for (let i = 0; i < forks_list.length; i++) { + forkNames.push(forks_list[i]["fork_name"]) } - return fork_names + return forkNames } const fetchForks = useCallback(async (repo) => { console.log('repo1',repo1); console.log('repo2', repo2); - const response = await getRepoForks(repo); - console.log("Fetching forks list for ", repo) - setData(getDataList(response.data.forks)) - setForkNames(getForkNames(response.data.forks)) + + //get total num of forks needs to be fetched + const active_fork_num = await getActiveForksNum(repo); + console.log("Active forks number is ", active_fork_num.data) + setActiveForksNum(active_fork_num.data) + + let total_list = [] + let counter = 0 + while (counter < active_fork_num.data) { + let res = await postProgress(repo, counter); + console.log(res.data.forks[0]) + total_list.push(res.data.forks[0]) + counter += 1 + setCounter(counter) + console.log(counter) + setProgress(counter/active_fork_num.data * 100) + console.log(progress) + } + console.log(total_list) + + // const response2 = await getProgress(repo); + // console.log("Fetching nnnnnnn for ", response2) + // const response = await getRepoForks(repo); + // console.log("Fetching forks list for ", repo) + // setForkList(response.data.forks); + setForkList(total_list); + // let data = getDailyCommit(response.data.forks) + let data = getDailyCommit(total_list) + let interval = Array.from(Array(28).keys()); + console.log(data) + setIntervalState(interval) + setData(data) + setForkNames(getForkNames(total_list)) }, []); useEffect(() => { @@ -53,7 +149,20 @@ const fetchForks = useCallback(async (repo) => { }, [fetchForks]); return ( - data && fork_names ? : + data && forkNames ? +
+ From feefac2bb55c3af6c7cd18159c94ab0b9eb5f532 Mon Sep 17 00:00:00 2001 From: Xinyan He Date: Sat, 22 Apr 2023 23:36:26 -0400 Subject: [PATCH 7/7] Add description for fork visualization --- app/api/FollowRepository.py | 3 --- client/src/FollowedRepositoryCard.jsx | 6 ++--- client/src/ForkGraph.jsx | 36 +++++++++++++++++++-------- client/src/ForkRank.jsx | 13 +++++----- client/src/Forklist.jsx | 1 + config.py | 2 +- 6 files changed, 37 insertions(+), 24 deletions(-) diff --git a/app/api/FollowRepository.py b/app/api/FollowRepository.py index e5e58b87..3abe21ee 100644 --- a/app/api/FollowRepository.py +++ b/app/api/FollowRepository.py @@ -71,9 +71,6 @@ def post(self): else: add_repo(_user.username, repo, res) db_followed_project(_user, repo) - print("lalalal zzzzzz") - r = db_find_project(repo) - print(r["analyser_progress"]) msg = ( "The repo (%s) starts loading into INFOX. We will send you an email when it is finished. Please wait." % repo diff --git a/client/src/FollowedRepositoryCard.jsx b/client/src/FollowedRepositoryCard.jsx index fe45019b..0abfd50b 100644 --- a/client/src/FollowedRepositoryCard.jsx +++ b/client/src/FollowedRepositoryCard.jsx @@ -56,10 +56,10 @@ const FollowedRepositoryCard = ({ expandIcon={} > - + {repo} - + - +