From b01b7065f2cdc06bf03fa1f1e9b888c4db1edebe Mon Sep 17 00:00:00 2001 From: jinhan Date: Thu, 28 Sep 2023 10:02:09 -0400 Subject: [PATCH 1/5] edit for h01 --- proofreadingprogress/app/common.py | 48 ++- proofreadingprogress/app/scripts/publish.js | 143 ++++---- proofreadingprogress/app/scripts/query.js | 344 ++++++++++-------- .../app/templates/publish.html | 10 +- proofreadingprogress/app/templates/query.html | 10 +- proofreadingprogress/app/templates/user.html | 10 +- run_dev.py | 6 +- 7 files changed, 315 insertions(+), 256 deletions(-) diff --git a/proofreadingprogress/app/common.py b/proofreadingprogress/app/common.py index dd5d5a1..2d63275 100644 --- a/proofreadingprogress/app/common.py +++ b/proofreadingprogress/app/common.py @@ -42,6 +42,7 @@ def index(): from .. import __version__ return f"ProofreadingProgress v{__version__}" + def query(): return render_template("query.html", prefix=__url_prefix__) @@ -65,6 +66,7 @@ def table(): def getResource(name): return send_from_directory(".", name) + def home(): resp = make_response() resp.headers["Access-Control-Allow-Origin"] = "*" @@ -136,14 +138,17 @@ def unhandled_exception(e): ) return 500 + # ------------------- # ------ Applications # ------------------- serverAddresses = { - "fly_v31": "https://prod.flywire-daf.com", - "fly_v26": "https://prod.flywire-daf.com", - "fly_training_v2": "https://minnie.microns-daf.com", + "h01_full0_v2": "https://local.brain-wire-test.org", + "test0_parents_v0": "https://local.brain-wire-test.org", + "fly_v26": "https://prod.flywire-daf.com" } + + def dataRequest(r): reqs = [] graph = None @@ -151,21 +156,21 @@ def dataRequest(r): raw = json.loads(r.data) single = args.get("query") isFiltered = args.get("filtered", "false") == "true" - isLineage = False#args.get("lineage", "false") == "true" + isLineage = False # args.get("lineage", "false") == "true" dataset = args.get("dataset", "default") - #use user token, instead of local token - client = chunkedgraph.ChunkedGraphClient(server_address=serverAddresses[dataset], - table_name=dataset, - auth_client=auth.AuthClient(token=g.auth_token)) - #print(f"My current token is: {client.auth.token}") + # use user token, instead of local token + client = chunkedgraph.ChunkedGraphClient(server_address=serverAddresses[dataset], + table_name=dataset, + auth_client=auth.AuthClient(token=g.auth_token)) + # print(f"My current token is: {auth.token}") str_queries = raw.get("queries", "").split(",") - queries = list(set(convertValidRootIds([single] if single else str_queries))) - + queries = list(set(convertValidRootIds( + [single] if single else str_queries))) dictsBatched = multiThread(client, queries, isFiltered) dfdict = {k: v for d in dictsBatched for k, v in d.items()} if (isLineage): graphsBatched = multiThread(client, queries, graph=True) - #graphs = [g for batch in graphsBatched for g in batch] + # graphs = [g for batch in graphsBatched for g in batch] batches = [batch for batch in graphsBatched] graphs = [g for g in batches] graph = nx.compose_all(graphs) if len(graphs) > 1 else graphs[0] @@ -183,14 +188,17 @@ def dataRequest(r): "json": reqs, } -def multiThread(client, queries, filter = True, graph = False, b_size = 10, p_size = 10): - bqueries = [(client, queries[i : i + b_size], filter, i) for i in range(0, len(queries), b_size)] + +def multiThread(client, queries, filter=True, graph=False, b_size=10, p_size=10): + bqueries = [(client, queries[i: i + b_size], filter, i) + for i in range(0, len(queries), b_size)] p = Pool(p_size) results = p.imap(caveGRPH if graph else caveCHLG, bqueries) p.close() p.join() return results + def caveCHLG(args): try: return args[0].get_tabular_change_log(args[1], args[2]) @@ -203,9 +211,9 @@ def caveCHLG(args): except: results[f'error_{args[3]}'].append(id) return results - -#return list of graphs + +# return list of graphs def caveGRPH(args): try: return args[0].get_lineage_graph(root_id=args[1], as_nx_graph=True) @@ -219,6 +227,7 @@ def caveGRPH(args): results[f'error_{args[3]}'].append(id) return results + def processToJson(query, dataframe, graph=None): pubdict = None published = [] @@ -242,6 +251,7 @@ def processToJson(query, dataframe, graph=None): "published": pubdict, } + def publish_neurons(args): if (True == True): return {} @@ -279,6 +289,7 @@ def publish_neurons(args): return existing + def removeInvalidRootIds(ids): valid = [] for id in ids: @@ -289,6 +300,7 @@ def removeInvalidRootIds(ids): pass return valid + def convertValidRootIds(ids): valid = [] for id in ids: @@ -298,17 +310,21 @@ def convertValidRootIds(ids): pass return valid + def validDOI(doi): valid = not len(doi) or re.match("^10.\d{4,9}[-._;()/:A-Z0-9]+$", doi) return doi if valid else "" + def validPaper(pname): valid = not len(pname) or re.match("^[\w\-\s]+$", pname) return pname if valid else "" + def publishRequest(args): return pd.DataFrame.from_dict(publish_neurons(args), orient="index").to_html() + def publishDump(): with engine.connect() as conn: return tableDump(conn).to_html() diff --git a/proofreadingprogress/app/scripts/publish.js b/proofreadingprogress/app/scripts/publish.js index 6b8f925..64b7796 100644 --- a/proofreadingprogress/app/scripts/publish.js +++ b/proofreadingprogress/app/scripts/publish.js @@ -1,147 +1,158 @@ const base = `${window.location.origin}/${ - document.getElementById('prefix').innerText || ''}/api/v1`; -const params = (new URL(document.location)).searchParams; -const auto_rootid = params.get('rootid'); + document.getElementById("prefix").innerText || "" +}/api/v1`; +const params = new URL(document.location).searchParams; +const auto_rootid = params.get("rootid"); const wparams = `location=no,toolbar=no,menubar=no,width=620,left=0,top=0`; -const fly_v31 = - `https://prodv1.flywire-daf.com/segmentation/api/v1/table/fly_v31/`; -const fly_training_v2 = - `https://minnie.microns-daf.com/segmentation/api/v1/table/fly_training_v2/`; +// const fly_v31 = +// `https://prodv1.flywire-daf.com/segmentation/api/v1/table/fly_v31/`; +// const fly_training_v2 = +// `https://minnie.microns-daf.com/segmentation/api/v1/table/fly_training_v2/`; +const h01_full0_v2 = `https://local.brain-wire-test.org/segmentation/1.0/h01_full0_v2/`; +const test0_parents_v0 = `https://local.brain-wire-test.org/segmentation/1.0/test0_parents_v0/`; const app = new Vue({ - el: '#app', + el: "#app", data: { // INPUT - dataset: fly_v31, - doi: '', - pname: '', - str_rootids: auto_rootid || '', + // dataset: fly_v31, + dataset: h01_full0_v2, + doi: "", + pname: "", + str_rootids: auto_rootid || "", // OUTPUT response: [], headers: [], - csv: '', + csv: "", // IMPORT colChoices: [], keyindex: 0, - importedCSVName: '', + importedCSVName: "", importedCSVFile: [], idToRowMap: {}, - status: 'Submit', - loading: false + status: "Submit", + loading: false, }, computed: { - isReady: function() { + isReady: function () { return this.str_rootids.length && this.validateAll() && !this.loading; }, - customDataset: function() { - return [fly_v31, fly_training_v2].includes(this.dataset); + customDataset: function () { + // return [fly_v31, fly_training_v2].includes(this.dataset); + return [h01_full0_v2, test0_parents_v0].includes(this.dataset); }, - validRoots: function() { + validRoots: function () { const valid = this.rootsIDTest(); return { - 'form-error': !valid, valid - } + "form-error": !valid, + valid, + }; }, - validDOI: function() { + validDOI: function () { const valid = this.DOITest(); return { - 'form-error': !valid, valid - } + "form-error": !valid, + valid, + }; }, - validTitle: function() { + validTitle: function () { const valid = this.titleTest(); return { - 'form-error': !valid, valid - } - } + "form-error": !valid, + valid, + }; + }, }, methods: { - validateAll: function() { + validateAll: function () { return this.rootsIDTest() && this.DOITest() && this.titleTest(); }, - rootsIDTest: function() { - return /^ *\d+ *(?:, *\d+ *)*$/gm.test(this.str_rootids) || - !this.str_rootids.length; + rootsIDTest: function () { + return ( + /^ *\d+ *(?:, *\d+ *)*$/gm.test(this.str_rootids) || + !this.str_rootids.length + ); }, - DOITest: function() { - return /^10.\d{4,9}[-._;()/:A-Z0-9]+$/i.test(this.doi) || - !this.doi.length; + DOITest: function () { + return ( + /^10.\d{4,9}[-._;()/:A-Z0-9]+$/i.test(this.doi) || !this.doi.length + ); }, - titleTest: function() { + titleTest: function () { return /^[\w\-\s]+$/.test(this.pname) || !this.pname.length; }, - apiRequest: async function() { + apiRequest: async function () { // Disable button, activate spinner this.loading = true; - this.status = 'Loading...'; + this.status = "Loading..."; this.processMQR(); const request = new URL(`${base}/pub/`); - request.searchParams.set('queries', this.str_rootids); - request.searchParams.set('verify', this.verify); - request.searchParams.set('doi', this.doi); - request.searchParams.set('pname', this.pname); - let path = new URL(this.dataset).pathname.split('/'); + request.searchParams.set("queries", this.str_rootids); + request.searchParams.set("verify", this.verify); + request.searchParams.set("doi", this.doi); + request.searchParams.set("pname", this.pname); + let path = new URL(this.dataset).pathname.split("/"); path.pop(); - request.searchParams.set('dataset', path.pop()); + request.searchParams.set("dataset", path.pop()); try { const response = await fetch(request); await this.processData(await response.json()); - this.status = 'Submit'; + this.status = "Submit"; this.loading = false; } catch (e) { alert(e); this.loading = false; - this.status = 'Submit'; + this.status = "Submit"; throw e; } }, - processData: function(response) { + processData: function (response) { this.queryProcess(response); }, - queryProcess: function(data) { + queryProcess: function (data) { let rows = Object.values(data); - rows.forEach(f => { + rows.forEach((f) => { if (this.headers.length == 0) { this.headers = Object.keys(f); } this.response = [...this.response, Object.values(f)]; }); }, - exportCSV: function() { - const filename = 'edits.csv'; - const blob = new Blob([this.csv], {type: 'text/csv;charset=utf-8;'}); - const link = document.createElement('a'); + exportCSV: function () { + const filename = "edits.csv"; + const blob = new Blob([this.csv], {type: "text/csv;charset=utf-8;"}); + const link = document.createElement("a"); const url = URL.createObjectURL(blob); - link.setAttribute('href', url); - link.setAttribute('download', filename); + link.setAttribute("href", url); + link.setAttribute("download", filename); link.click(); }, - importCSV: function(e) { + importCSV: function (e) { Papa.parse(e.target.files[0], { skipEmptyLines: true, complete: (results) => { this.importedCSVName = e.target.files[0]; this.importedCSVFile = results.data; this.colChoices = results.data[0]; - } + }, }); }, - chooseCSV: function() { - document.getElementById('import').click(); + chooseCSV: function () { + document.getElementById("import").click(); }, - importCol: function(index) { + importCol: function (index) { this.importedCSVFile.forEach((e, i) => { this.keyindex = index; // Ignore first row (header) if (i) { let rid = e[index]; - if (rid[0] == '\'' && rid.length > 1) rid = rid.slice(1); - this.str_rootids = this.str_rootids.concat(i == 1 ? '' : ', ', rid); + if (rid[0] == "'" && rid.length > 1) rid = rid.slice(1); + this.str_rootids = this.str_rootids.concat(i == 1 ? "" : ", ", rid); this.idToRowMap[rid] = i; } }); - } - } -}); \ No newline at end of file + }, + }, +}); diff --git a/proofreadingprogress/app/scripts/query.js b/proofreadingprogress/app/scripts/query.js index c4216b9..1b59dcd 100644 --- a/proofreadingprogress/app/scripts/query.js +++ b/proofreadingprogress/app/scripts/query.js @@ -1,12 +1,16 @@ const base = `${window.location.origin}/${ - document.getElementById('prefix').innerText || ''}/api/v1`; -const params = (new URL(document.location)).searchParams; -const auto_rootid = params.get('rootid'); -const auto_submit = params.get('submit'); -const auto_dataset = params.get('dataset'); + document.getElementById("prefix").innerText || "" +}/api/v1`; +const params = new URL(document.location).searchParams; +const auto_rootid = params.get("rootid"); +const auto_submit = params.get("submit"); +const auto_dataset = params.get("dataset"); const wparams = `location=no,toolbar=no,menubar=no,width=620,left=0,top=0`; -document.querySelectorAll('#info').forEach( - e => e.addEventListener('click', v => v.stopImmediatePropagation())); +document + .querySelectorAll("#info") + .forEach((e) => + e.addEventListener("click", (v) => v.stopImmediatePropagation()) + ); // HELPERS function percent(num) { var m = Number((Math.abs(num) * 10000).toPrecision(15)); @@ -15,12 +19,11 @@ function percent(num) { function clrPopup(popup, data) { try { popup.app = null; - } catch { - } + } catch {} } function partition(items, size) { var p = []; - for (var i = Math.ceil(items.length / size); i-- > 0;) { + for (var i = Math.ceil(items.length / size); i-- > 0; ) { p[i] = items.slice(i * size, (i + 1) * size); } return p; @@ -30,27 +33,29 @@ function setPopup(popup, data) { popup.app.$data.headers = data.headers; popup.app.$data.response = data.response; } catch { - setTimeout(() => {setPopup(popup, data)}, 250); + setTimeout(() => { + setPopup(popup, data); + }, 250); } } -function openWindowWithGet(url, data, name = '', params = '') { +function openWindowWithGet(url, data, name = "", params = "") { // var usp = new URLSearchParams(data); // var popup = window.open(url, name, params); - var popup = window.open(url, name); //, params); + var popup = window.open(url, name); //, params); clrPopup(popup, data); setPopup(popup, data); } function openWindowWithPost(url, data) { - var form = document.createElement('form'); - form.target = '_blank'; + var form = document.createElement("form"); + form.target = "_blank"; // TODO: modify to work better - form.method = 'POST'; + form.method = "POST"; form.action = url; - form.style.display = 'none'; + form.style.display = "none"; for (var key in data) { - var input = document.createElement('input'); - input.type = 'hidden'; + var input = document.createElement("input"); + input.type = "hidden"; input.name = key; input.value = data[key]; form.appendChild(input); @@ -60,147 +65,158 @@ function openWindowWithPost(url, data) { document.body.removeChild(form); } function importCSVCheck(id) { - return /^ *\d+ *(?:, *\d+ *)*$/gm.test(id) ? id : 0 -}; + return /^ *\d+ *(?:, *\d+ *)*$/gm.test(id) ? id : 0; +} const app = new Vue({ - el: '#app', + el: "#app", mounted() { if (auto_submit) this.apiRequest(); }, data: { // INPUT - query: {root_id: auto_rootid || '', filtered: true, lineage: true}, + query: {root_id: auto_rootid || "", filtered: true, lineage: true}, excelcsv: false, - dataset: auto_dataset || 'fly_v31', - str_multiquery: '', + dataset: auto_dataset || "h01_full0_v2", + str_multiquery: "", // OUTPUT - error: '', - warn: '', + error: "", + warn: "", failedIDs: [], - failedIDString: '', + failedIDString: "", response: [], headers: [], - csv: '', - userCSV: '', - ex_csv: '', + csv: "", + userCSV: "", + ex_csv: "", userList: {}, userHeaders: [], // IMPORT colChoices: [], keyindex: 0, - importedCSVName: '', + importedCSVName: "", importedCSVFile: [], importOverride: 0, idToRowMap: {}, // ETC - status: 'Submit', - loading: false + status: "Submit", + loading: false, }, computed: { - isReady: function() { - return (this.query.root_id.length && - !isNaN(parseInt(this.query.root_id)) || - this.str_multiquery.length) && - this.rootsIDTest() && !this.loading; + isReady: function () { + return ( + ((this.query.root_id.length && !isNaN(parseInt(this.query.root_id))) || + this.str_multiquery.length) && + this.rootsIDTest() && + !this.loading + ); }, - validRoots: function() { + validRoots: function () { const valid = this.rootsIDTest(); return { - 'form-error': !valid, valid - } + "form-error": !valid, + valid, + }; + }, + rootCount: function () { + return !this.str_multiquery.length + ? 0 + : this.str_multiquery.split(/[ ,]+/).length; }, - rootCount: function() { - return !this.str_multiquery.length ? - 0 : - this.str_multiquery.split(/[ ,]+/).length; - } }, methods: { - rootsIDTest: function() { - return /^ *\d+ *(?:, *\d+ *)*$/gm.test(this.str_multiquery) || - /^ *\d+ *(?: +\d+ *)*$/gm.test(this.str_multiquery) || - !this.str_multiquery.length; + rootsIDTest: function () { + return ( + /^ *\d+ *(?:, *\d+ *)*$/gm.test(this.str_multiquery) || + /^ *\d+ *(?: +\d+ *)*$/gm.test(this.str_multiquery) || + !this.str_multiquery.length + ); }, - apiRequest: async function() { + apiRequest: async function () { // Disable button, activate spinner this.loading = true; - this.status = 'Loading...'; - + this.status = "Loading..."; + console.log(base); const request = new URL(`${base}/qry/`); if (!this.query.root_id.length) { // request.searchParams.set('queries', this.str_multiquery); } else { - request.searchParams.set('query', this.query.root_id); + request.searchParams.set("query", this.query.root_id); } - request.searchParams.set('dataset', this.dataset); - request.searchParams.set('filtered', this.query.filtered); - request.searchParams.set('lineage', this.query.lineage); + request.searchParams.set("dataset", this.dataset); + request.searchParams.set("filtered", this.query.filtered); + request.searchParams.set("lineage", this.query.lineage); const responses = []; const rawRequests = this.str_multiquery.split(/[ ,\n]+/); const requests = Array.from(new Set(rawRequests)); const dupCount = rawRequests.length - requests.length; - this.warn = dupCount ? `${dupCount} duplicates removed.` : ''; + this.warn = dupCount ? `${dupCount} duplicates removed.` : ""; const batches = partition(requests, 100); let failed = 0; for (const reqSet of batches) { try { // Reset to zero this.failedIDs = []; - this.failedIDString = ''; + this.failedIDString = ""; this.error = ``; - - responses.push(await fetch(request, { - method: 'POST', - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify({queries: reqSet.join(',')}), - })); + console.log(request); + responses.push( + await fetch(request, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({queries: reqSet.join(",")}), + }) + ); } catch (e) { this.failedIDs = [...this.failedIDs, ...reqSet]; - this.failedIDString = this.failedIDs.join(', '); - this.error = - `Could not communicate with server, please retry the following IDs:\n`; + this.failedIDString = this.failedIDs.join(", "); + this.error = `Could not communicate with server, please retry the following IDs:\n`; } } + console.log(responses); try { - const resArray = - await Promise.all(responses.map(async r => await r.json())); - const jsonArray = resArray.map(d => d.json).flat(); - const errorArray = resArray.map(d => d.error).flat(); - if (this.error == '' && errorArray.length) { + const resArray = await Promise.all( + responses.map(async (r) => await r.json()) + ); + const jsonArray = resArray.map((d) => d.json).flat(); + const errorArray = resArray.map((d) => d.error).flat(); + if (this.error == "" && errorArray.length) { this.failedIDs = errorArray; - this.failedIDString = errorArray.join(', '); + this.failedIDString = errorArray.join(", "); this.error = `The following ids could not be processed.`; } console.log( - `${requests.length} requests, ${jsonArray.length} responses`); + `${requests.length} requests, ${jsonArray.length} responses` + ); await this.processData(jsonArray); - this.status = 'Submit'; + this.status = "Submit"; this.loading = false; } catch (e) { this.loading = false; - this.status = 'Submit'; + this.status = "Submit"; this.warn = e; throw e; } }, - processData: function(rawData) { + processData: function (rawData) { const singleRow = rawData[0]; if (!this.str_multiquery.length && rawData[0]) { if (singleRow.edits.length) { this.headers = Object.keys(singleRow.edits[0]); - singleRow.edits.forEach(row => { + singleRow.edits.forEach((row) => { if (row.timestamp) { row.timestamp = new Date(row.timestamp).toUTCString(); } }); this.response = singleRow.edits; - const csv = - [this.headers, ...this.response.map(e => Object.values(e))]; + const csv = [ + this.headers, + ...this.response.map((e) => Object.values(e)), + ]; this.csv = Papa.unparse(csv); } else { this.warn = `Root ID ${this.query.root_id} has no edits`; @@ -209,7 +225,7 @@ const app = new Vue({ this.queryProcess(rawData); } }, - queryProcess: function(data) { + queryProcess: function (data) { /* In lieu of aggreation on server/batching of request * Assume user_id always present* Array of SegmentIdResponse Arrays => UserIDMap where each UserID has @@ -219,11 +235,11 @@ const app = new Vue({ const segmentList = []; data.forEach((seg, i) => { const id = seg.key; - seg.edits.forEach(f => { + seg.edits.forEach((f) => { if (!userIdsHash[f.user_id]) { userIdsHash[f.user_id] = { user_id: f.user_id, - user_name: f.user_name + user_name: f.user_name, }; } if (!userIdsHash[f.user_id][id]) { @@ -235,7 +251,8 @@ const app = new Vue({ } }); const segMapRow = [ - ['segment_ID', id], ['total_edits', seg.edits.length], + ["segment_ID", id], + ["total_edits", seg.edits.length], //['published', seg.published], ]; if (this.query.lineage) { @@ -245,49 +262,56 @@ const app = new Vue({ }); this.generateResponse(segmentList, userIdsHash); }, - generateResponse: function(segments, uids) { - this.userHeaders = - ['user_name', 'user_id', 'total_edits', 'contributed>10%']; + generateResponse: function (segments, uids) { + this.userHeaders = [ + "user_name", + "user_id", + "total_edits", + "contributed>10%", + ]; let userList = {}; - this.response = segments.map(seg => { + this.response = segments.map((seg) => { let headers = Array.from(seg.keys()); let row = Array.from(seg.values()); - seg.set('contributor', []); + seg.set("contributor", []); - let segId = seg.get('segment_ID'); - let isPublished = seg.get('published'); + let segId = seg.get("segment_ID"); + let isPublished = seg.get("published"); // there may be a better way to sort (requires filtering) let contributor = Object.keys(uids) - .filter(a => uids[a][segId]) - .sort( - (a, b) => uids[b][segId].number_of_edits - - uids[a][segId].number_of_edits); + .filter((a) => uids[a][segId]) + .sort( + (a, b) => + uids[b][segId].number_of_edits - uids[a][segId].number_of_edits + ); - contributor.forEach(uid => { + contributor.forEach((uid) => { let user = uids[uid]; let contributed = user[segId]; let edits = - contributed && !isPublished ? contributed.number_of_edits : 0; + contributed && !isPublished ? contributed.number_of_edits : 0; - let percentEdits = percent((edits / seg.get('total_edits'))); + let percentEdits = percent(edits / seg.get("total_edits")); let contributor = new Map([ - ['user_name', user.user_name], ['user_id', uid], - ['number_of_edits', edits], ['percent_of_total', percentEdits] + ["user_name", user.user_name], + ["user_id", uid], + ["number_of_edits", edits], + ["percent_of_total", percentEdits], ]); headers = headers.concat(Array.from(contributor.keys())); row = row.concat(Array.from(contributor.values())); - seg.get('contributor').push(contributor); + seg.get("contributor").push(contributor); if (!userList[uid]) { userList[uid] = { user_name: user.user_name, user_id: uid, total_edits: 0, - contributed_at_least_10Percent: 0 + contributed_at_least_10Percent: 0, }; } userList[uid].total_edits += edits; userList[uid].contributed_at_least_10Percent += - ((percentEdits >= 10) ? 1 : 0); + percentEdits >= 10 ? 1 : 0; }); if (this.headers.length < headers.length) { this.headers = headers; @@ -299,28 +323,31 @@ const app = new Vue({ this.csv = Papa.unparse(csv); this.userCSV = Papa.unparse(this.userList); - const ex_res = this.response.map(row => { - var newRow = [...row] + const ex_res = this.response.map((row) => { + var newRow = [...row]; newRow[0] = `'${newRow[0]}`; return newRow; }); const ex_csv = [this.headers, ...ex_res]; this.ex_csv = Papa.unparse(ex_csv); }, - exportCSV: function() { - rawExport('edits.csv', this.excelcsv ? this.ex_csv : this.csv); + exportCSV: function () { + rawExport("edits.csv", this.excelcsv ? this.ex_csv : this.csv); }, - exportSingleCSV: function() { - rawExport('changelog.csv', this.csv); + exportSingleCSV: function () { + rawExport("changelog.csv", this.csv); }, - viewResults: function() { + viewResults: function () { let headers = this.headers; let response = this.response; openWindowWithGet( - new URL(`${base}/table`), {headers, response, select: true}, - `Edits Table`, wparams); + new URL(`${base}/table`), + {headers, response, select: true}, + `Edits Table`, + wparams + ); }, - mergeCSV: function() { + mergeCSV: function () { const filename = `${this.importedCSVName.name}_merged.csv`; const mergedCSV = [...this.importedCSVFile]; Papa.parse(this.csv, { @@ -331,32 +358,38 @@ const app = new Vue({ // if (!i) return; // const key = this.idToRowMap[row[0]]; const firsthalf = mergedCSV[key].slice(0, this.keyindex); - const secondhalf = this.keyindex < mergedCSV[key].length - 1 ? - mergedCSV[key].slice(this.keyindex + 1) : - []; + const secondhalf = + this.keyindex < mergedCSV[key].length - 1 + ? mergedCSV[key].slice(this.keyindex + 1) + : []; mergedCSV[key] = [...firsthalf, ...secondhalf, ...row]; }); - const blob = new Blob( - [Papa.unparse(mergedCSV)], {type: 'text/csv;charset=utf-8;'}); - const link = document.createElement('a'); + const blob = new Blob([Papa.unparse(mergedCSV)], { + type: "text/csv;charset=utf-8;", + }); + const link = document.createElement("a"); const url = URL.createObjectURL(blob); - link.setAttribute('href', url); - link.setAttribute('download', filename); + link.setAttribute("href", url); + link.setAttribute("download", filename); link.click(); - } + }, }); }, - exportUserCSV: function() { - rawExport('useredits.csv', this.userCSV); + exportUserCSV: function () { + rawExport("useredits.csv", this.userCSV); }, - viewUsers: function() { + viewUsers: function () { let headers = this.userHeaders; - let response = this.userList.map(r => Object.values(r)); + let response = this.userList.map((r) => Object.values(r)); openWindowWithGet( - new URL(`${base}/table`), {headers, response}, `User Table`, wparams); + new URL(`${base}/table`), + {headers, response}, + `User Table`, + wparams + ); }, - importCSV: function(e) { + importCSV: function (e) { Papa.parse(e.target.files[0], { // quoteChar: `'`, // escapeChar: `\\`, @@ -368,21 +401,21 @@ const app = new Vue({ if (!results.data.length) { this.warn = `${this.importedCSVName} is empty!`; } else if ( - results.data.length > 1 && - results.data[0].length != results.data[1].length) { - this.warn = - `Header Row and Row 1 have different amount of columns! Unable to import columns correctly. + results.data.length > 1 && + results.data[0].length != results.data[1].length + ) { + this.warn = `Header Row and Row 1 have different amount of columns! Unable to import columns correctly. Please ensure that the imported CSV is valid. Every row must have the same amount of commas.`; } this.colChoices = results.data[0]; - } + }, }); }, - chooseCSV: function() { - document.getElementById('import').click(); + chooseCSV: function () { + document.getElementById("import").click(); }, - importCol: function(index) { + importCol: function (index) { const multiquery = []; let badRows = 0; this.importedCSVFile.forEach((e, i) => { @@ -390,7 +423,7 @@ const app = new Vue({ this.keyindex = index; let rid = e[index]; // Remove Leading ' on import - if (rid[0] == '\'' && rid.length > 1) rid = rid.slice(1); + if (rid[0] == "'" && rid.length > 1) rid = rid.slice(1); // Remove invalid root_ids rid = importCSVCheck(rid); if (rid) { @@ -399,29 +432,28 @@ const app = new Vue({ } } catch { badRows++; - this.warn = - `${badRows} rows could not be imported. Index out of bounds.` + this.warn = `${badRows} rows could not be imported. Index out of bounds.`; } }); - this.str_multiquery = !multiquery.length ? '' : multiquery.join(', '); + this.str_multiquery = !multiquery.length ? "" : multiquery.join(", "); }, - overrideImport: function() { + overrideImport: function () { this.importCol(Number(this.importOverride)); - } - } + }, + }, }); -$(function() { - $('[data-toggle="tooltip"]').tooltip() -}) +$(function () { + $('[data-toggle="tooltip"]').tooltip(); +}); function rawExport(name, csv) { const filename = name; - const blob = new Blob([csv], {type: 'text/csv;charset=utf-8;'}); - const link = document.createElement('a'); + const blob = new Blob([csv], {type: "text/csv;charset=utf-8;"}); + const link = document.createElement("a"); const url = URL.createObjectURL(blob); - link.setAttribute('href', url); - link.setAttribute('download', filename); + link.setAttribute("href", url); + link.setAttribute("download", filename); link.click(); -} \ No newline at end of file +} diff --git a/proofreadingprogress/app/templates/publish.html b/proofreadingprogress/app/templates/publish.html index 270f2c1..a949dc8 100644 --- a/proofreadingprogress/app/templates/publish.html +++ b/proofreadingprogress/app/templates/publish.html @@ -25,11 +25,11 @@ diff --git a/proofreadingprogress/app/templates/query.html b/proofreadingprogress/app/templates/query.html index 0aa321b..3cb1868 100644 --- a/proofreadingprogress/app/templates/query.html +++ b/proofreadingprogress/app/templates/query.html @@ -31,11 +31,11 @@
Segment ID:
diff --git a/proofreadingprogress/app/templates/user.html b/proofreadingprogress/app/templates/user.html index 0cc4954..004f2ef 100644 --- a/proofreadingprogress/app/templates/user.html +++ b/proofreadingprogress/app/templates/user.html @@ -61,11 +61,11 @@
diff --git a/run_dev.py b/run_dev.py index 410d016..ed969a6 100644 --- a/run_dev.py +++ b/run_dev.py @@ -8,9 +8,9 @@ app = create_app() -os.environ["INFO_URL"] = "global.daf-apis.com/info" -os.environ["AUTH_URL"] = "global.daf-apis.com/auth" -os.environ["STICKY_AUTH_URL"] = "global.daf-apis.com/sticky_auth" +os.environ["INFO_URL"] = "global.brain-wire-test.org/info" +os.environ["AUTH_URL"] = "global.brain-wire-test.org/auth" +os.environ["STICKY_AUTH_URL"] = "global.brain-wire-test.org/sticky_auth" if __name__ == "__main__": print(sys.argv) From a34c84f47c0b28d350364c12759b4426fdea8217 Mon Sep 17 00:00:00 2001 From: jinhan Date: Fri, 29 Sep 2023 10:26:44 -0400 Subject: [PATCH 2/5] edit to use env variables & add env.sh for testing --- envs.sh | 3 +++ proofreadingprogress/app/common.py | 14 ++++++-------- proofreadingprogress/app/scripts/publish.js | 16 ++++++---------- proofreadingprogress/app/scripts/query.js | 5 ++++- proofreadingprogress/app/templates/publish.html | 13 ++++++------- proofreadingprogress/app/templates/query.html | 8 +++----- proofreadingprogress/app/templates/user.html | 8 +++----- 7 files changed, 31 insertions(+), 36 deletions(-) create mode 100644 envs.sh diff --git a/envs.sh b/envs.sh new file mode 100644 index 0000000..7e3b863 --- /dev/null +++ b/envs.sh @@ -0,0 +1,3 @@ +export PPROGRESS_DATASET="h01_full0_v2" +export DNS_HOSTNAME="https://local.brain-wire-test.org" +export PPROGRESS_DATASET_URL=export PPROGRESS_DATASET_URL='https://local.brain-wire-test.org/segmentation/1.0/h01_full0_v2/' diff --git a/proofreadingprogress/app/common.py b/proofreadingprogress/app/common.py index 2d63275..f9d46fc 100644 --- a/proofreadingprogress/app/common.py +++ b/proofreadingprogress/app/common.py @@ -22,7 +22,8 @@ __api_versions__ = [0] __url_prefix__ = os.environ.get("PPROGRESS_URL_PREFIX", "progress") - +pprogress_dataset = os.environ.get("PPROGRESS_DATASET") +pprogressDatasetURL = os.environ.get("PPROGRESS_DATASET_URL") # auth_token_file = open( # os.path.join( # os.path.expanduser("~"), ".cloudvolume/secrets/chunkedgraph-secret.json" @@ -44,11 +45,11 @@ def index(): def query(): - return render_template("query.html", prefix=__url_prefix__) + return render_template("query.html", prefix=__url_prefix__, pprogressDataset=pprogress_dataset) def user(): - return render_template("user.html", prefix=__url_prefix__) + return render_template("user.html", prefix=__url_prefix__, pprogressDataset=pprogress_dataset) def base(): @@ -56,7 +57,7 @@ def base(): def publish(): - return render_template("publish.html", prefix=__url_prefix__) + return render_template("publish.html", prefix=__url_prefix__, pprogressDataset=pprogress_dataset, pprogressDatasetURL=pprogressDatasetURL) def table(): @@ -143,12 +144,9 @@ def unhandled_exception(e): # ------ Applications # ------------------- serverAddresses = { - "h01_full0_v2": "https://local.brain-wire-test.org", - "test0_parents_v0": "https://local.brain-wire-test.org", - "fly_v26": "https://prod.flywire-daf.com" + os.environ.get("PPROGRESS_DATASET"): os.environ.get("DNS_HOSTNAME") } - def dataRequest(r): reqs = [] graph = None diff --git a/proofreadingprogress/app/scripts/publish.js b/proofreadingprogress/app/scripts/publish.js index 64b7796..542297f 100644 --- a/proofreadingprogress/app/scripts/publish.js +++ b/proofreadingprogress/app/scripts/publish.js @@ -4,19 +4,16 @@ const base = `${window.location.origin}/${ const params = new URL(document.location).searchParams; const auto_rootid = params.get("rootid"); const wparams = `location=no,toolbar=no,menubar=no,width=620,left=0,top=0`; -// const fly_v31 = -// `https://prodv1.flywire-daf.com/segmentation/api/v1/table/fly_v31/`; -// const fly_training_v2 = -// `https://minnie.microns-daf.com/segmentation/api/v1/table/fly_training_v2/`; -const h01_full0_v2 = `https://local.brain-wire-test.org/segmentation/1.0/h01_full0_v2/`; -const test0_parents_v0 = `https://local.brain-wire-test.org/segmentation/1.0/test0_parents_v0/`; +const pprogressDataset = + document.getElementById("pprogressDataset").innerText || ""; +const pprogressDatasetURL = + document.getElementById("pprogressDatasetURL").innerText || ""; const app = new Vue({ el: "#app", data: { // INPUT - // dataset: fly_v31, - dataset: h01_full0_v2, + dataset: pprogressDatasetURL, doi: "", pname: "", str_rootids: auto_rootid || "", @@ -38,8 +35,7 @@ const app = new Vue({ return this.str_rootids.length && this.validateAll() && !this.loading; }, customDataset: function () { - // return [fly_v31, fly_training_v2].includes(this.dataset); - return [h01_full0_v2, test0_parents_v0].includes(this.dataset); + return [pprogressDataset].includes(this.dataset); }, validRoots: function () { const valid = this.rootsIDTest(); diff --git a/proofreadingprogress/app/scripts/query.js b/proofreadingprogress/app/scripts/query.js index 1b59dcd..86b79cc 100644 --- a/proofreadingprogress/app/scripts/query.js +++ b/proofreadingprogress/app/scripts/query.js @@ -6,6 +6,9 @@ const auto_rootid = params.get("rootid"); const auto_submit = params.get("submit"); const auto_dataset = params.get("dataset"); const wparams = `location=no,toolbar=no,menubar=no,width=620,left=0,top=0`; +const pprogressDataset = + document.getElementById("pprogressDataset").innerText || ""; + document .querySelectorAll("#info") .forEach((e) => @@ -77,7 +80,7 @@ const app = new Vue({ // INPUT query: {root_id: auto_rootid || "", filtered: true, lineage: true}, excelcsv: false, - dataset: auto_dataset || "h01_full0_v2", + dataset: auto_dataset || pprogressDataset, str_multiquery: "", // OUTPUT error: "", diff --git a/proofreadingprogress/app/templates/publish.html b/proofreadingprogress/app/templates/publish.html index a949dc8..59d10fe 100644 --- a/proofreadingprogress/app/templates/publish.html +++ b/proofreadingprogress/app/templates/publish.html @@ -15,7 +15,11 @@ crossorigin="anonymous"> - + + + + +
@@ -25,12 +29,7 @@ diff --git a/proofreadingprogress/app/templates/query.html b/proofreadingprogress/app/templates/query.html index 3cb1868..54a92d1 100644 --- a/proofreadingprogress/app/templates/query.html +++ b/proofreadingprogress/app/templates/query.html @@ -31,11 +31,8 @@
Segment ID:
@@ -44,6 +41,7 @@
Segment ID:
+
diff --git a/proofreadingprogress/app/templates/user.html b/proofreadingprogress/app/templates/user.html index 004f2ef..f0356fb 100644 --- a/proofreadingprogress/app/templates/user.html +++ b/proofreadingprogress/app/templates/user.html @@ -61,16 +61,14 @@
+
@@ -29,7 +27,9 @@ @@ -98,6 +98,10 @@
+ diff --git a/proofreadingprogress/app/templates/query.html b/proofreadingprogress/app/templates/query.html index 54a92d1..d11b068 100644 --- a/proofreadingprogress/app/templates/query.html +++ b/proofreadingprogress/app/templates/query.html @@ -31,8 +31,9 @@
Segment ID:
@@ -41,7 +42,6 @@
Segment ID:
- @@ -56,5 +56,8 @@
Segment ID:
+ {% endblock %} \ No newline at end of file diff --git a/proofreadingprogress/app/templates/user.html b/proofreadingprogress/app/templates/user.html index f0356fb..bcbb49d 100644 --- a/proofreadingprogress/app/templates/user.html +++ b/proofreadingprogress/app/templates/user.html @@ -61,14 +61,14 @@
-