merge concurrent-download
parent
f407c235c7
commit
f028c332f3
|
@ -5,6 +5,8 @@
|
||||||
import NumpyLoader from "../utils/loadnpy";
|
import NumpyLoader from "../utils/loadnpy";
|
||||||
import * as CanApi from "../api/can";
|
import * as CanApi from "../api/can";
|
||||||
|
|
||||||
|
const MAX_CONNECTIONS = 8;
|
||||||
|
|
||||||
var window = self;
|
var window = self;
|
||||||
|
|
||||||
const Int64LE = require("int64-buffer").Int64LE;
|
const Int64LE = require("int64-buffer").Int64LE;
|
||||||
|
@ -16,30 +18,51 @@ async function fetchAndPostData(
|
||||||
[minPart, maxPart],
|
[minPart, maxPart],
|
||||||
canStartTime
|
canStartTime
|
||||||
) {
|
) {
|
||||||
console.log("\n\nfetchAndPostData", `${currentPart} of ${maxPart}`);
|
console.log("starting fetchAndPostData process");
|
||||||
|
|
||||||
// if we've exhausted the parts, close up shop
|
var partList = [];
|
||||||
if (currentPart > maxPart) {
|
var minPart = currentPart;
|
||||||
self.postMessage({
|
var prevPart = Promise.resolve();
|
||||||
progress: 100,
|
var promiseBuffer = [];
|
||||||
shouldClose: true
|
var totalParts = maxPart - minPart;
|
||||||
});
|
var currentProcess = 0;
|
||||||
self.close();
|
while (currentPart <= maxPart) {
|
||||||
return;
|
// post inc, pass the previous value then inc
|
||||||
}
|
console.log("Starting download", currentPart - minPart, "of", totalParts);
|
||||||
|
prevPart = downloadNextPart(
|
||||||
let awaitedData = null;
|
|
||||||
try {
|
|
||||||
awaitedData = await CanApi.fetchCanPart(base, currentPart);
|
|
||||||
} catch (e) {
|
|
||||||
console.log("fetchCanPart missing part", e);
|
|
||||||
return fetchAndPostData(
|
|
||||||
base,
|
base,
|
||||||
currentPart + 1,
|
canStartTime,
|
||||||
[minPart, maxPart],
|
currentProcess,
|
||||||
canStartTime
|
prevPart,
|
||||||
|
currentPart++
|
||||||
);
|
);
|
||||||
|
currentProcess = ~~(100 * ((currentPart - minPart) / totalParts));
|
||||||
|
|
||||||
|
promiseBuffer.push(prevPart);
|
||||||
|
if (promiseBuffer.length > MAX_CONNECTIONS) {
|
||||||
|
await promiseBuffer.shift();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await prevPart;
|
||||||
|
// processing is done!
|
||||||
|
self.postMessage({
|
||||||
|
progress: 100,
|
||||||
|
shouldClose: true
|
||||||
|
});
|
||||||
|
self.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function downloadNextPart(
|
||||||
|
base,
|
||||||
|
canStartTime,
|
||||||
|
currentProcess,
|
||||||
|
prevPartPromise,
|
||||||
|
currentPart
|
||||||
|
) {
|
||||||
|
var awaitedData = await CanApi.fetchCanPart(base, currentPart);
|
||||||
|
await prevPartPromise;
|
||||||
|
|
||||||
const { times, sources, addresses, datas } = awaitedData;
|
const { times, sources, addresses, datas } = awaitedData;
|
||||||
|
|
||||||
// times is a float64array, which we want to be a normal array for now
|
// times is a float64array, which we want to be a normal array for now
|
||||||
|
@ -62,12 +85,12 @@ async function fetchAndPostData(
|
||||||
|
|
||||||
console.log("posting message");
|
console.log("posting message");
|
||||||
self.postMessage({
|
self.postMessage({
|
||||||
progress: 10,
|
progress: currentProcess,
|
||||||
logData,
|
logData,
|
||||||
shouldClose: false
|
shouldClose: false
|
||||||
});
|
});
|
||||||
|
|
||||||
fetchAndPostData(base, currentPart + 1, [minPart, maxPart], canStartTime);
|
return awaitedData;
|
||||||
}
|
}
|
||||||
|
|
||||||
function transformAndSend(rawData) {
|
function transformAndSend(rawData) {
|
||||||
|
|
Loading…
Reference in New Issue