如何将 ContinuationToken 与递归文件夹迭代器一起使用

Question

由于云端硬盘 API 配额、服务配额和脚本执行时间限制6 min因此将 Google 云端硬盘文件操作拆分为区块通常至关重要。

我们可以使用属性服务来存储文件夹迭代器或文件迭代器的continuationToken。 这样，我们可以停止脚本，并在下次运行时从停止的位置继续。

工作示例(线性迭代器(

// Logs the name of every file in the User's Drive
// this is useful as the script may take more that 5 minutes (max execution time)
var userProperties = PropertiesService.getUserProperties();
var continuationToken = userProperties.getProperty('CONTINUATION_TOKEN');
var start = new Date();
var end = new Date();
var maxTime = 1000*60*4.5; // Max safe time, 4.5 mins
if (continuationToken == null) {
// firt time execution, get all files from Drive
var files = DriveApp.getFiles();
} else {
// not the first time, pick up where we left off
var files = DriveApp.continueFileIterator(continuationToken);
}
while (files.hasNext() && end.getTime() - start.getTime() <= maxTime) {
var file = files.next();
Logger.log(file.getName());
end = new Date();
}
// Save your place by setting the token in your user properties
if(files.hasNext()){
var continuationToken = files.getContinuationToken();
userProperties.setProperty('CONTINUATION_TOKEN', continuationToken);
} else {
// Delete the token
PropertiesService.getUserProperties().deleteProperty('CONTINUATION_TOKEN');
}

问题(递归迭代器(

为了检索文件夹的树状结构并获取其文件，我们必须使用递归函数。像这样

：

doFolders(DriveApp.getFolderById('root folder id'));
// recursive iteration
function doFolders(parentFolder) {
var childFolders = parentFolder.getFolders();
while(childFolders.hasNext()) {
var child = childFolders.next();
// do something with folder
// go subfolders
doFolders(child);
}
}

但是，在这种情况下，我不知道如何使用continuationToken.

问题

当我们需要抛出所有文件夹结构时，如何将ContinuationToken与递归文件夹迭代器一起使用？

假设

根据每个parent文件夹的id构造许多带有名称的令牌是否有意义？

Answer 1

如果尝试递归迭代文件夹并想要使用延续标记(大型文件夹可能需要(，则需要一个可以存储多组继续标记的数据结构。既适用于文件和文件夹，也适用于当前层次结构中的每个文件夹。

最简单的数据结构是对象数组。

下面是一个解决方案，它为您提供了用于创建函数的模板，该函数可以递归处理文件并存储继续标记，以便在超时时可以恢复。

1. 只需将MAX_RUNNING_TIME_MS修改为所需的值(现在设置为 1 分钟(。

   您不希望将其设置超过 ~4.9 分钟，因为脚本可能会在此之前超时并且不存储其当前状态。

2. 更新processFile方法以对文件执行任何操作。
3. 最后，调用processRootFolder()并传递Folder。它将足够聪明，知道如何继续处理文件夹。

当然还有改进的余地(例如，它只是检查文件夹名称以查看它是恢复还是重新启动(，但这很可能足以满足 95% 需要递归迭代具有继续令牌的文件夹的人。

function processRootFolder(rootFolder) {
var MAX_RUNNING_TIME_MS = 1 * 60 * 1000;
var RECURSIVE_ITERATOR_KEY = "RECURSIVE_ITERATOR_KEY";
var startTime = (new Date()).getTime();
// [{folderName: String, fileIteratorContinuationToken: String?, folderIteratorContinuationToken: String}]
var recursiveIterator = JSON.parse(PropertiesService.getDocumentProperties().getProperty(RECURSIVE_ITERATOR_KEY));
if (recursiveIterator !== null) {
// verify that it's actually for the same folder
if (rootFolder.getName() !== recursiveIterator[0].folderName) {
console.warn("Looks like this is a new folder. Clearing out the old iterator.");
recursiveIterator = null;
} else {
console.info("Resuming session.");
}
}
if (recursiveIterator === null) {
console.info("Starting new session.");
recursiveIterator = [];
recursiveIterator.push(makeIterationFromFolder(rootFolder));
}
while (recursiveIterator.length > 0) {
recursiveIterator = nextIteration(recursiveIterator, startTime);
var currTime = (new Date()).getTime();
var elapsedTimeInMS = currTime - startTime;
var timeLimitExceeded = elapsedTimeInMS >= MAX_RUNNING_TIME_MS;
if (timeLimitExceeded) {
PropertiesService.getDocumentProperties().setProperty(RECURSIVE_ITERATOR_KEY, JSON.stringify(recursiveIterator));
console.info("Stopping loop after '%d' milliseconds. Please continue running.", elapsedTimeInMS);
return;
}
}
console.info("Done running");
PropertiesService.getDocumentProperties().deleteProperty(RECURSIVE_ITERATOR_KEY);
}
// process the next file or folder
function nextIteration(recursiveIterator) {
var currentIteration = recursiveIterator[recursiveIterator.length-1];
if (currentIteration.fileIteratorContinuationToken !== null) {
var fileIterator = DriveApp.continueFileIterator(currentIteration.fileIteratorContinuationToken);
if (fileIterator.hasNext()) {
// process the next file
var path = recursiveIterator.map(function(iteration) { return iteration.folderName; }).join("/");
processFile(fileIterator.next(), path);
currentIteration.fileIteratorContinuationToken = fileIterator.getContinuationToken();
recursiveIterator[recursiveIterator.length-1] = currentIteration;
return recursiveIterator;
} else {
// done processing files
currentIteration.fileIteratorContinuationToken = null;
recursiveIterator[recursiveIterator.length-1] = currentIteration;
return recursiveIterator;
}
}
if (currentIteration.folderIteratorContinuationToken !== null) {
var folderIterator = DriveApp.continueFolderIterator(currentIteration.folderIteratorContinuationToken);
if (folderIterator.hasNext()) {
// process the next folder
var folder = folderIterator.next();
recursiveIterator[recursiveIterator.length-1].folderIteratorContinuationToken = folderIterator.getContinuationToken();
recursiveIterator.push(makeIterationFromFolder(folder));
return recursiveIterator;
} else {
// done processing subfolders
recursiveIterator.pop();
return recursiveIterator;
}
}
throw "should never get here";
}
function makeIterationFromFolder(folder) {
return {
folderName: folder.getName(), 
fileIteratorContinuationToken: folder.getFiles().getContinuationToken(),
folderIteratorContinuationToken: folder.getFolders().getContinuationToken()
};
}
function processFile(file, path) {
console.log(path + "/" + file.getName());
}