diff --git a/.gitignore b/.gitignore index cc01764..281995a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /unpackage/dist /unpackage/cache -/.vscode \ No newline at end of file +/.vscode +/node_modules +package-lock.json \ No newline at end of file diff --git a/oss-plugin.js b/oss-plugin.js new file mode 100644 index 0000000..a3c2d93 --- /dev/null +++ b/oss-plugin.js @@ -0,0 +1,295 @@ +const fs = require('fs') +const path = require('path') +const OSS = require('ali-oss') +const globby = require('globby') +const Listr = require('listr') +require('colors') + +class WebpackAliyunOss { + constructor(options) { + const { region, accessKeyId, accessKeySecret, bucket, secure = false } = options + + this.config = Object.assign( + { + test: false, // 测试 + dist: '', // oss目录 + buildRoot: '.', // 构建目录名 + deleteOrigin: false, // 是否删除源文件 + timeout: 30 * 1000, // 超时时间 + parallel: 5, // 并发数 + setOssPath: null, // 手动设置每个文件的上传路径 + setHeaders: null, // 设置头部 + overwrite: false, // 覆盖oss同名文件 + bail: false, // 出错中断上传 + logToLocal: false // 出错信息写入本地文件 + }, + options + ) + + this.configErrStr = this.checkOptions(options) + + this.client = new OSS({ + region, + accessKeyId, + accessKeySecret, + bucket, + secure + }) + + this.filesUploaded = [] + this.filesIgnored = [] + } + + apply(compiler) { + if (compiler) { + return this.doWithWebpack(compiler) + } else { + return this.doWidthoutWebpack() + } + } + + doWithWebpack(compiler) { + compiler.hooks.afterEmit.tapPromise('WebpackAliyunOss', async (compilation) => { + if (this.configErrStr) { + compilation.errors.push(this.configErrStr) + return Promise.resolve() + } + + const outputPath = path.resolve(this.slash(compiler.options.output.path)) + + const { from = outputPath + '/**' } = this.config + + const files = await globby(from) + + if (files.length) { + try { + await this.upload(files, true, outputPath) + console.log('') + console.log(' All files uploaded successfully '.bgGreen.bold.white) + } catch (err) { + compilation.errors.push(err) + return Promise.reject(err) + } + } else { + console.log('no files to be uploaded') + return Promise.resolve('no files to be uploaded') + } + }) + } + + async doWidthoutWebpack() { + if (this.configErrStr) return Promise.reject(this.configErrStr) + + const { from } = this.config + const files = await globby(from) + + if (files.length) { + try { + await this.upload(files) + console.log('') + console.log(' All files uploaded successfully '.bgGreen.bold.white) + } catch (err) { + return Promise.reject(err) + } + } else { + console.log('no files to be uploaded') + return Promise.resolve('no files to be uploaded') + } + } + + async upload(files, inWebpack, outputPath = '') { + const { dist, setHeaders, deleteOrigin, setOssPath, timeout, test, overwrite, bail, parallel, logToLocal } = this.config + + if (test) { + console.log('') + console.log("Currently running in test mode. your files won't realy be uploaded.".green.underline) + console.log('') + } else { + console.log('') + console.log('Your files will be uploaded very soon.'.green.underline) + console.log('') + } + + files = files.map((file) => ({ + path: file, + fullPath: path.resolve(file) + })) + + this.filesUploaded = [] + this.filesIgnored = [] + this.filesErrors = [] + + const basePath = this.getBasePath(inWebpack, outputPath) + + const _upload = async (file) => { + const { fullPath: filePath, path: fPath } = file + + let ossFilePath = this.slash(path.join(dist, (setOssPath && setOssPath(filePath)) || (basePath && filePath.split(basePath)[1]) || '')) + + if (test) { + return Promise.resolve(fPath.blue.underline + ' is ready to upload to ' + ossFilePath.green.underline) + } + + if (!overwrite) { + const fileExists = await this.fileExists(ossFilePath) + if (fileExists) { + this.filesIgnored.push(filePath) + return Promise.resolve(fPath.blue.underline + ' ready exists in oss, ignored') + } + } + + const headers = (setHeaders && setHeaders(filePath)) || {} + let result + try { + result = await this.client.put(ossFilePath, filePath, { + timeout, + // headers: !overwrite ? Object.assign(headers, { 'x-oss-forbid-overwrite': true }) : headers + headers + }) + } catch (err) { + // if (err.name === 'FileAlreadyExistsError') { + // this.filesIgnored.push(filePath) + // return Promise.resolve(fPath.blue.underline + ' ready exists in oss, ignored'); + // } + + this.filesErrors.push({ + file: fPath, + err: { code: err.code, message: err.message, name: err.name } + }) + + const errorMsg = `Failed to upload ${fPath.underline}: ` + `${err.name}-${err.code}: ${err.message}`.red + return Promise.reject(new Error(errorMsg)) + } + + result.url = this.normalize(result.url) + this.filesUploaded.push(fPath) + + if (deleteOrigin) { + fs.unlinkSync(filePath) + this.deleteEmptyDir(filePath) + } + + return Promise.resolve(fPath.blue.underline + ' successfully uploaded, oss url => ' + result.url.green) + } + + let len = parallel + const addTask = () => { + if (len < files.length) { + tasks.add(createTask(files[len])) + len++ + } + } + const createTask = (file) => ({ + title: `uploading ${file.path.underline}`, + task(_, task) { + return _upload(file) + .then((msg) => { + task.title = msg + addTask() + }) + .catch((e) => { + if (!bail) addTask() + return Promise.reject(e) + }) + } + }) + const tasks = new Listr(files.slice(0, len).map(createTask), { + exitOnError: bail, + concurrent: parallel + }) + + await tasks.run().catch(() => {}) + + // this.filesIgnored.length && console.log('files ignored due to not overwrite'.blue, this.filesIgnored); + + if (this.filesErrors.length) { + console.log(' UPLOAD ENDED WITH ERRORS '.bgRed.white, '\n') + logToLocal && fs.writeFileSync(path.resolve('upload.error.log'), JSON.stringify(this.filesErrors, null, 2)) + + return Promise.reject(' UPLOAD ENDED WITH ERRORS ') + } + } + + getBasePath(inWebpack, outputPath) { + if (this.config.setOssPath) return '' + + let basePath = '' + + if (inWebpack) { + if (path.isAbsolute(outputPath)) basePath = outputPath + else basePath = path.resolve(outputPath) + } else { + const { buildRoot } = this.config + if (path.isAbsolute(buildRoot)) basePath = buildRoot + else basePath = path.resolve(buildRoot) + } + + return this.slash(basePath) + } + + fileExists(filepath) { + // return this.client.get(filepath) + return this.client + .head(filepath) + .then((result) => { + return result.res.status == 200 + }) + .catch((e) => { + if (e.code == 'NoSuchKey') return false + }) + } + + normalize(url) { + const tmpArr = url.split(/\/{2,}/) + if (tmpArr.length >= 2) { + const [protocol, ...rest] = tmpArr + url = protocol + '//' + rest.join('/') + } + return url + } + + slash(path) { + const isExtendedLengthPath = /^\\\\\?\\/.test(path) + // const hasNonAscii = /[^\u0000-\u0080]+/.test(path); + + if (isExtendedLengthPath) { + return path + } + + return path.replace(/\\/g, '/') + } + + deleteEmptyDir(filePath) { + let dirname = path.dirname(filePath) + if (fs.existsSync(dirname) && fs.statSync(dirname).isDirectory()) { + fs.readdir(dirname, (err, files) => { + if (err) console.error(err) + else { + if (!files.length) fs.rmdir(dirname, () => {}) + } + }) + } + } + + checkOptions(options = {}) { + const { from, region, accessKeyId, accessKeySecret, bucket } = options + + let errStr = '' + + if (!region) errStr += '\nregion not specified' + if (!accessKeyId) errStr += '\naccessKeyId not specified' + if (!accessKeySecret) errStr += '\naccessKeySecret not specified' + if (!bucket) errStr += '\nbucket not specified' + + if (Array.isArray(from)) { + if (from.some((g) => typeof g !== 'string')) errStr += '\neach item in from should be a glob string' + } else { + let fromType = typeof from + if (['undefined', 'string'].indexOf(fromType) === -1) errStr += '\nfrom should be string or array' + } + + return errStr + } +} + +module.exports = WebpackAliyunOss diff --git a/oss.config.json b/oss.config.json new file mode 100644 index 0000000..a06bc10 --- /dev/null +++ b/oss.config.json @@ -0,0 +1,5 @@ +{ + "buildDir": "unpackage/dist/build/h5", + "bucket": "paper-shopkeeper-app", + "ignoreFiles": ["MP_verify_4rGTG1rVNG0UhMkH.txt"] +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..7171d08 --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "oss-upload", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "push:test": "node pushOss.js test", + "push:prod": "node pushOss.js prod" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "ali-oss": "^6.10.0", + "colors": "^1.3.2", + "fs-extra": "^8.1.0", + "globby": "^10.0.1", + "listr": "^0.14.3" + } +} diff --git a/pushOss.js b/pushOss.js new file mode 100644 index 0000000..74f31e2 --- /dev/null +++ b/pushOss.js @@ -0,0 +1,73 @@ +// const WebpackAliyunOss = require('webpack-aliyun-oss') +const WebpackAliyunOss = require('./oss-plugin.js') +// 注意上传文件中不要有包含buildDir的文件名 +let { buildDir, bucket, ignoreFiles } = require('./oss.config.json') +const args = process.argv.slice(2) || [] + +if (!buildDir || !bucket) { + console.log('请配置 oss.config.json') + process.exit(1) +} + +const [env] = args + +bucket = bucket + (env === 'prod' ? '' : `-${env}`) +console.log('bucket:', bucket) + +const oss = { + region: 'oss-cn-shenzhen', + accessKeyId: 'LTAINmC91NqIGN38', + accessKeySecret: 'Hh10dQPjq1jMLLSpbDAR05ZzR3nXsU', + bucket: bucket +} + +const instance = new WebpackAliyunOss({ + from: [`./${buildDir}/**`], + dist: '/', + region: oss.region, + accessKeyId: oss.accessKeyId, + accessKeySecret: oss.accessKeySecret, + bucket: oss.bucket, + overwrite: true, + secure: true, + timeout: 300000, // 5分钟 + setOssPath(filePath) { + const fp = filePath.replace(/\\/g, '/') + const index = fp.lastIndexOf(buildDir) + const Path = fp.substring(index + buildDir.length, filePath.length) + return Path + }, + setHeaders(filePath) { + return { + //'Cache-Control': 'max-age=31536000' + } + } +}) + +let client = instance.client + +async function deleteAll() { + const { objects } = await client.list() + if (objects.length) { + let files = objects.map((item) => item.name) + if (ignoreFiles.length > 0) { + files = files.filter((item) => !ignoreFiles.includes(item)) + } + let count = 0 + while (count < files.length) { + // 一次最多1000个文件 + await client.deleteMulti(files.slice(count, count + 1000), { + quiet: true + }) + count += 1000 + } + } + console.log('bucket 文件删除完成') +} + +async function run() { + await deleteAll() + await instance.apply() +} + +run()