Skip to content

Commit 64ab389

Browse files
randyourandyou
authored
Merge pull request #14 from randyou/master
支持上传文件夹
2 parents d34583c + 804027f commit 64ab389

File tree

4 files changed

+158
-34
lines changed

4 files changed

+158
-34
lines changed

‎bin/index.js‎

Lines changed: 82 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ const path = require('path');
88
require('winston-daily-rotate-file');
99
const ProgressBar = require('progress');
1010
const BlueBirdPromise = require("bluebird");
11+
const glob = require('glob');
1112

1213
const logger = require('../lib/log');
1314
const { DEFAULT_CHUNK_SIZE, MAX_CHUNK } = require('../lib/constants');
@@ -33,7 +34,7 @@ process.on('uncaughtException', error => {
3334
logger.error(error.stack);
3435
})
3536

36-
const upload = async (filePath, parts = []) => {
37+
const upload = async (filePath, parts = [], requestUrl) => {
3738
const bar = new ProgressBar(':bar [:current/:total] :percent ', { total: totalChunk });
3839
const uploadChunk = async (currentChunk, currentChunkIndex, parts, isRetry) => {
3940
if (parts.some(({ partNumber, size }) => partNumber === currentChunkIndex && size === currentChunk.length)) {
@@ -47,7 +48,7 @@ const upload = async (filePath, parts = []) => {
4748
version,
4849
partNumber: currentChunkIndex,
4950
size: currentChunk.length,
50-
currentChunk
51+
currentChunk
5152
}, {
5253
headers: {
5354
'Content-Type': 'application/octet-stream'
@@ -75,14 +76,14 @@ const upload = async (filePath, parts = []) => {
7576
}
7677
}
7778

78-
console.log(`\n开始上传\n`)
79-
logger.info('开始上传')
79+
console.log(`\n开始上传 (${filePath})\n`);
80+
logger.info(`开始上传 (${filePath})`);
8081

8182
try {
8283

83-
const chunkIndexs = new Array(totalChunk).fill("").map((_,index) => index+1)
84+
const chunkIndexs = new Array(totalChunk).fill("").map((_, index) => index + 1)
8485

85-
await BlueBirdPromise.map(chunkIndexs,(currentChunkIndex)=>{
86+
await BlueBirdPromise.map(chunkIndexs, (currentChunkIndex) => {
8687
const start = (currentChunkIndex - 1) * chunkSize;
8788
const end = ((start + chunkSize) >= fileSize) ? fileSize : start + chunkSize - 1;
8889
const stream = fs.createReadStream(filePath, { start, end })
@@ -113,9 +114,9 @@ const upload = async (filePath, parts = []) => {
113114

114115

115116

116-
117117

118-
const merge = async () => {
118+
119+
const merge = async () => {
119120
console.log(chalk.cyan('正在合并分片，请稍等...'))
120121
return await _mergeAllChunks(requestUrl, {
121122
version,
@@ -126,7 +127,7 @@ const upload = async (filePath, parts = []) => {
126127
Authorization
127128
});
128129
}
129-
130+
130131

131132
try {
132133
const res = await withRetry(merge, 3, 500);
@@ -140,11 +141,11 @@ const upload = async (filePath, parts = []) => {
140141
return;
141142
}
142143

143-
console.log(chalk.green(`\n上传完毕\n`))
144+
console.log(chalk.green(`\n上传完毕 (${filePath})\n`))
144145
logger.info('************************ 上传完毕 ************************')
145146
}
146147

147-
const getFileMD5Success = async (filePath) => {
148+
const getFileMD5Success = async (filePath, requestUrl) => {
148149
try {
149150
const res = await _getExistChunks(requestUrl, {
150151
fileSize,
@@ -160,10 +161,10 @@ const getFileMD5Success = async (filePath) => {
160161

161162
// 上传过一部分
162163
if (Array.isArray(res.data.parts)) {
163-
await upload(filePath, res.data.parts);
164+
await upload(filePath, res.data.parts, requestUrl);
164165
} else {
165166
// 未上传过
166-
await upload(filePath);
167+
await upload(filePath, [], requestUrl);
167168
}
168169
} catch (error) {
169170
logger.error(error.message);
@@ -173,20 +174,20 @@ const getFileMD5Success = async (filePath) => {
173174
}
174175
}
175176

176-
const getFileMD5 = async (filePath) => {
177+
const getFileMD5 = async (filePath, requestUrl) => {
177178
totalChunk = Math.ceil(fileSize / DEFAULT_CHUNK_SIZE);
178179
if (totalChunk > MAX_CHUNK) {
179180
chunkSize = Math.ceil(fileSize / MAX_CHUNK);
180181
totalChunk = Math.ceil(fileSize / chunkSize);
181182
}
182183
const spark = new SparkMD5.ArrayBuffer();
183184
try {
184-
console.log(`\n开始计算 MD5\n`)
185-
logger.info('开始计算 MD5')
185+
console.log(`\n开始计算 MD5 (${filePath})\n`);
186+
logger.info(`开始计算 MD5 (${filePath})`);
186187

187188
const bar = new ProgressBar(':bar [:current/:total] :percent ', { total: totalChunk });
188189
await new Promise(resolve => {
189-
stream = fs.createReadStream(filePath, { highWaterMark: chunkSize })
190+
stream = fs.createReadStream(filePath, { highWaterMark: chunkSize });
190191
stream.on('data', chunk => {
191192
bar.tick();
192193
spark.append(chunk)
@@ -198,7 +199,7 @@ const getFileMD5 = async (filePath) => {
198199
md5 = spark.end();
199200
spark.destroy();
200201
console.log(`\n文件 MD5：${md5}\n`)
201-
await getFileMD5Success(filePath);
202+
await getFileMD5Success(filePath, requestUrl);
202203
resolve();
203204
})
204205
}).catch(error => {
@@ -212,14 +213,70 @@ const getFileMD5 = async (filePath) => {
212213
}
213214
}
214215

216+
const uploadFile = async (filePath, size, requestUrl) => {
217+
fileSize = size;
218+
await getFileMD5(filePath, requestUrl);
219+
md5 = '';
220+
uploadId = '';
221+
fileSize = 0;
222+
chunkSize = DEFAULT_CHUNK_SIZE;
223+
totalChunk = 0;
224+
}
225+
226+
const uploadDir = async (dir) => {
227+
let files = [];
228+
try {
229+
files = await new Promise((resolve, reject) => {
230+
glob("**/**", {
231+
cwd: dir,
232+
root: dir
233+
}, function (error, files = []) {
234+
if (error) {
235+
reject(error);
236+
} else {
237+
resolve(files)
238+
}
239+
})
240+
});
241+
} catch (error) {
242+
if (error) {
243+
console.log(chalk.red((error.response && error.response.data) || error.message));
244+
logger.error(error.message);
245+
logger.error(error.stack);
246+
process.exit(1);
247+
} else {
248+
resolve(files)
249+
}
250+
}
251+
252+
253+
for (const file of files) {
254+
const filePath = path.join(dir, file);
255+
const stat = fs.lstatSync(filePath);
256+
const isDirectory = stat.isDirectory();
257+
if (!isDirectory) {
258+
const url = new URL(`chunks/${dir.split(path.sep).pop()}/${file}`, requestUrl.endsWith('/') ? requestUrl : `${requestUrl}/`).toString();
259+
await uploadFile(filePath, stat.size, url);
260+
console.log('************************ **** ************************');
261+
logger.info('************************ **** ************************');
262+
}
263+
}
264+
}
265+
215266
const beforeUpload = async (filePath) => {
267+
const isUploadDir = argv.dir;
268+
let fSize = 0;
216269
try {
217270
const stat = fs.lstatSync(filePath);
218-
if (stat.isDirectory()) {
271+
const isDirectory = stat.isDirectory();
272+
if (isDirectory && !isUploadDir) {
219273
console.log(chalk.red(`\n${filePath}不合法，需指定一个文件\n`))
220274
process.exit(1);
275+
} else if (!isDirectory && isUploadDir) {
276+
console.log(chalk.red(`\n${filePath}不合法，需指定一个文件夹\n`))
277+
process.exit(1);
221278
}
222-
fileSize = stat.size;
279+
fSize = stat.size;
223280
} catch (error) {
224281
if (error.code === 'ENOENT') {
225282
console.log(chalk.red(`未找到 ${filePath}`));
@@ -230,7 +287,11 @@ const beforeUpload = async (filePath) => {
230287
}
231288
process.exit(1);
232289
}
233-
await getFileMD5(filePath);
290+
if (isUploadDir) {
291+
await uploadDir(filePath);
292+
} else {
293+
await uploadFile(filePath, fSize, requestUrl);
294+
}
234295
}
235296

236297
const onUpload = (_username, _password) => {

‎lib/argv.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const argv = require('yargs')
2-
.usage('用法: coding-generic --username=<USERNAME>[:PASSWORD] --path=<FILE.EXT> --registry=<REGISTRY>')
2+
.usage('上传文件: coding-generic --username=<USERNAME>[:PASSWORD] --path=<FILE.EXT> --registry=<REGISTRY>')
3+
.usage('上传文件夹: coding-generic --username=<USERNAME>[:PASSWORD] --dir --path=<FOLDER> --registry=<REGISTRY>')
34
.options({
45
username: {
56
alias: 'u',
@@ -21,12 +22,18 @@ const argv = require('yargs')
2122
describe: '上传分块并行数',
2223
demandOption: true,
2324
default: 5,
25+
},
26+
dir: {
27+
alias: 'd',
28+
describe: '上传文件夹',
29+
boolean: true,
2430
}
2531
})
2632
.alias('version', 'v')
2733
.help('h')
2834
.alias('h', 'help')
29-
.example('coding-generic --username=coding@coding.com:123456 --path=./test.txt --registry="https://codingcorp-generic.pkg.coding.net/project-name/generic-repo/chunks/test.txt?version=latest"')
35+
.example('上传文件: coding-generic --username=coding@coding.com:123456 --path=./test.txt --registry="https://codingcorp-generic.pkg.coding.net/project-name/generic-repo/chunks/test.txt?version=latest"')
36+
.example('上传文件夹: coding-generic --username=coding@coding.com:123456 --dir --path=./dirname --registry="https://codingcorp-generic.pkg.coding.net/project-name/generic-repo?version=latest"')
3037
.argv;
3138

3239
module.exports = argv;

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "coding-generic",
3-
"version": "1.2.6",
3+
"version": "1.2.7",
44
"description": "",
55
"main": "index.js",
66
"bin": {
@@ -15,6 +15,7 @@
1515
"chalk": "^4.1.0",
1616
"cos-nodejs-sdk-v5": "^2.8.2",
1717
"form-data": "^3.0.0",
18+
"glob": "^8.0.3",
1819
"progress": "^2.0.3",
1920
"prompts": "^2.3.2",
2021
"spark-md5": "^3.0.1",

‎yarn.lock‎

Lines changed: 65 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,17 @@ aws4@^1.8.0:
7070
resolved "https://registry.npm.taobao.org/aws4/download/aws4-1.10.1.tgz?cache=0&sync_timestamp=1597236947743&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faws4%2Fdownload%2Faws4-1.10.1.tgz#e1e82e4f3e999e2cfd61b161280d16a111f86428"
7171
integrity sha1-4eguTz6Zniz9YbFhKA0WoRH4ZCg=
7272

73-
axios@^0.20.0:
74-
version "0.20.0"
75-
resolved "https://registry.npm.taobao.org/axios/download/axios-0.20.0.tgz#057ba30f04884694993a8cd07fa394cff11c50bd"
76-
integrity sha1-BXujDwSIRpSZOozQf6OUz/EcUL0=
73+
axios@^0.21.1:
74+
version "0.21.4"
75+
resolved "https://mirrors.tencent.com/npm/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
76+
integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
7777
dependencies:
78-
follow-redirects "^1.10.0"
78+
follow-redirects "^1.14.0"
79+
80+
balanced-match@^1.0.0:
81+
version "1.0.2"
82+
resolved "https://mirrors.tencent.com/npm/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
83+
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
7984

8085
bcrypt-pbkdf@^1.0.0:
8186
version "1.0.2"
@@ -89,6 +94,13 @@ bluebird@^3.7.2:
8994
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
9095
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
9196

97+
brace-expansion@^2.0.1:
98+
version "2.0.1"
99+
resolved "https://mirrors.tencent.com/npm/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
100+
integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
101+
dependencies:
102+
balanced-match "^1.0.0"
103+
92104
caseless@~0.12.0:
93105
version "0.12.0"
94106
resolved "https://registry.npm.taobao.org/caseless/download/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@@ -310,10 +322,10 @@ fn.name@1.x.x:
310322
resolved "https://registry.npm.taobao.org/fn.name/download/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
311323
integrity sha1-JsrYAXlnrqhzG8QpYdBKPVmIrMw=
312324

313-
follow-redirects@^1.10.0:
314-
version "1.13.0"
315-
resolved "https://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.13.0.tgz?cache=0&sync_timestamp=1597057976909&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ffollow-redirects%2Fdownload%2Ffollow-redirects-1.13.0.tgz#b42e8d93a2a7eea5ed88633676d6597bc8e384db"
316-
integrity sha1-tC6Nk6Kn7qXtiGM2dtZZe8jjhNs=
325+
follow-redirects@^1.14.0:
326+
version "1.15.2"
327+
resolved "https://mirrors.tencent.com/npm/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
328+
integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
317329

318330
forever-agent@~0.6.1:
319331
version "0.6.1"
@@ -338,6 +350,11 @@ form-data@~2.3.2:
338350
combined-stream "^1.0.6"
339351
mime-types "^2.1.12"
340352

353+
fs.realpath@^1.0.0:
354+
version "1.0.0"
355+
resolved "https://mirrors.tencent.com/npm/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
356+
integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
357+
341358
get-caller-file@^2.0.5:
342359
version "2.0.5"
343360
resolved "https://registry.npm.taobao.org/get-caller-file/download/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
@@ -350,6 +367,17 @@ getpass@^0.1.1:
350367
dependencies:
351368
assert-plus "^1.0.0"
352369

370+
glob@^8.0.3:
371+
version "8.0.3"
372+
resolved "https://mirrors.tencent.com/npm/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e"
373+
integrity sha512-ull455NHSHI/Y1FqGaaYFaLGkNMMJbavMrEGFXG/PGrg6y7sutWHUHrz6gy6WEBH6akM1M414dWKCNs+IhKdiQ==
374+
dependencies:
375+
fs.realpath "^1.0.0"
376+
inflight "^1.0.4"
377+
inherits "2"
378+
minimatch "^5.0.1"
379+
once "^1.3.0"
380+
353381
har-schema@^2.0.0:
354382
version "2.0.0"
355383
resolved "https://registry.npm.taobao.org/har-schema/download/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
@@ -377,7 +405,15 @@ http-signature@~1.2.0:
377405
jsprim "^1.2.2"
378406
sshpk "^1.7.0"
379407

380-
inherits@^2.0.3, inherits@~2.0.3:
408+
inflight@^1.0.4:
409+
version "1.0.6"
410+
resolved "https://mirrors.tencent.com/npm/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
411+
integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=
412+
dependencies:
413+
once "^1.3.0"
414+
wrappy "1"
415+
416+
inherits@2, inherits@^2.0.3, inherits@~2.0.3:
381417
version "2.0.4"
382418
resolved "https://registry.npm.taobao.org/inherits/download/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
383419
integrity sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=
@@ -510,6 +546,13 @@ mimic-fn@^3.0.0:
510546
resolved "https://registry.npm.taobao.org/mimic-fn/download/mimic-fn-3.1.0.tgz?cache=0&sync_timestamp=1596095644798&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fmimic-fn%2Fdownload%2Fmimic-fn-3.1.0.tgz#65755145bbf3e36954b949c16450427451d5ca74"
511547
integrity sha1-ZXVRRbvz42lUuUnBZFBCdFHVynQ=
512548

549+
minimatch@^5.0.1:
550+
version "5.1.0"
551+
resolved "https://mirrors.tencent.com/npm/minimatch/-/minimatch-5.1.0.tgz#1717b464f4971b144f6aabe8f2d0b8e4511e09c7"
552+
integrity sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==
553+
dependencies:
554+
brace-expansion "^2.0.1"
555+
513556
moment@^2.11.2:
514557
version "2.29.1"
515558
resolved "https://registry.npm.taobao.org/moment/download/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
@@ -530,6 +573,13 @@ object-hash@^2.0.1:
530573
resolved "https://registry.npm.taobao.org/object-hash/download/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea"
531574
integrity sha1-0S2wROA80so9d8BXDYciWwLh5uo=
532575

576+
once@^1.3.0:
577+
version "1.4.0"
578+
resolved "https://mirrors.tencent.com/npm/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
579+
integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
580+
dependencies:
581+
wrappy "1"
582+
533583
one-time@^1.0.0:
534584
version "1.0.0"
535585
resolved "https://registry.npm.taobao.org/one-time/download/one-time-1.0.0.tgz#e06bc174aed214ed58edede573b433bbf827cb45"
@@ -868,6 +918,11 @@ wrap-ansi@^7.0.0:
868918
string-width "^4.1.0"
869919
strip-ansi "^6.0.0"
870920

921+
wrappy@1:
922+
version "1.0.2"
923+
resolved "https://mirrors.tencent.com/npm/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
924+
integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
925+
871926
xml2js@^0.4.19:
872927
version "0.4.23"
873928
resolved "https://registry.npm.taobao.org/xml2js/download/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"

0 commit comments

Comments
 (0)