Skip to content
Snippets Groups Projects
Commit 504a0e80 authored by Dean's avatar Dean
Browse files

added scanner logic and SQS stuff, untested

parent e9c959e3
No related branches found
No related tags found
No related merge requests found
{ {
"extends": "standard", "extends": "standard",
"installedESLint": true, "installedESLint": true,
"plugins": [ "plugins": [
"standard", "standard",
"promise" "promise"
], ],
"rules": { "rules": {
"semi": [2, "always"] "semi": [2, "always"]
} }
} }
MIT License MIT License
Copyright (c) 2016 OwO.Whats-Th.is? Copyright (c) 2016 Dean Sheather
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
......
module.exports = { // Required modules
getFile: require('./lib/getfile'), const clam = require('clamscan')();
scanner: require('./lib/scanner'), const crypto = require('crypto');
refreshclam: require('./lib/refreshclam') const freshclam = require('./lib/freshclam.js');
} const fs = require('fs');
\ No newline at end of file const path = require('path');
const S3 = require('./lib/S3.js');
const SQS = require('./lib/SQS.js');
const scanner = require('./lib/scanner.js');
// Check for required environment variables
for (let env of [
'AWS_ACCESSKEY',
'AWS_SECRETKEY',
'AWS_SQSURL'
]) {
if (!process.env.hasOwnProperty(env)) {
throw new Error(`missing required environment variable "${env}"`);
}
}
// TODO: loop freshclam
/**
* Handle all errors.
* @param {Error} error
*/
function handleError (error) {
// TODO: error handling
}
/**
* Check SQS for messages and process virus scanning.
*/
function pollSQS () {
SQS.receiveMessage({
MaxNumberOfMessages: 10,
QueueUrl: process.env['AWS_SQSURL'],
WaitTimeSeconds: 20
}, function (err, data) {
if (err) return handleError(error);
let promises = [];
(data.Records || []).forEach(msg => {
promises.push(
Promise.resolve(msg)
.then(msg => JSON.parse(msg.Body))
.then(body => { msg: msg, msgBody: body })
.then(getObject)
.then(writeTempFile)
.then(clamScan)
.then(unlinkTempFile)
// .then(fireNotification)
.then(deleteInfectedFromS3)
.then(deleteSQSMessage)
.catch(handleError)
);
});
// Loop
Promise.all(promises)
.then(() => setImmediate(pollSQS))
.catch(err => handleError(err) && setImmediate(pollSQS));
});
}
/**
* Get object from S3, promisified.
* @param {Object} params
* @return {Promise<Object, Error>}
*/
function getObject (data) {
return new Promise((resolve, reject) => {
S3.getObject({
Bucket: body.s3.bucket.name,
Key: body.s3.object.Key
}, (err, res) => {
if (err) return reject(err);
data.Body = new Buffer(res.Body);
resolve(data);
});
});
}
/**
* Generate random key.
* @return {string} 6 character key.
*/
function generateRandomKey () {
const seed = String(Math.floor(Math.random() * 10) + Date.now());
return crypto.createHash('md5').update(seed).digest('hex').substr(2, 6);
}
/**
* Create a temporary file on disk for scanning.
*/
function writeTempFile (data) {
return new Promise((resolve, reject) => {
// Construct the filepath (including random key)
const filepath = path.join('.', '_temp', data.Bucket, generateRandomKey() + data.Key.replace(/[^a-z0-9_.-]/gi, '_'));
// Write the file
fs.writeFile(filepath, data.Body, err => {
if (err) return reject(err);
data.filepath = filepath;
resolve(data);
});
});
}
/**
* Scan the file for viruses.
*/
function clamScan (data) {
return new Promise((resolve, reject) => {
clam.is_infected(data.filepath, (err, _, isInfected) => {
if (err) return reject(err);
data.isInfected = isInfected;
resolve(data);
});
});
}
/**
* Unlink temporary file.
*/
function unlinkTempFile (data) {
return new Promise((resolve, reject) => {
fs.unlink(filepath, (err) => {
if (err) return reject(err);
resolve(data);
});
});
}
/**
* Deleted infected files from S3.
*/
function deleteInfectedFromS3 (data) {
return new Promise((resolve, reject) => {
if (!data.isInfected) return resolve(data);
S3.deleteObject({ Bucket: data.Bucket, Key: data.Key }, (err, res) => {
if (err) return reject(err);
data.wasPermanentlyDeletedFromS3 = res.DeleteMarker;
resolve(data);
});
});
}
/**
* Delete processed SQS message.
*/
function deleteSQSMessage (data) {
return new Promise((resolve, reject) => {
SQS.deleteMessage({
QueueUrl: process.env['AWS_SQSURL'],
ReceiptHandle: data.msg.ReceiptHandle
}, (err) => {
if (err) return reject(err);
resolve(data);
});
});
}
// Required modules
const AWS = require('aws-sdk');
// Create S3 client
module.exports = new AWS.S3({
apiVersion: '2006-03-01',
accessKeyId: process.env['AWS_ACCESSKEY'],
secretAccessKey: process.env['AWS_SECRETKEY']
});
// Required modules
const AWS = require('aws-sdk');
// Create SQS client
module.exports = new AWS.SQS({
apiVersion: '2012-11-05',
accessKeyId: process.env['AWS_ACCESSKEY'],
secretAccessKey: process.env['AWS_SECRETKEY']
});
// Required modules
const exec = require('child_process').exec;
const debug = require('debug')('scanner:freshclam');
/**
* Run `freshclam` using child_process in order to refresh the ClamAV virus
* database on the system.
* @return {Promise<undefined, Error>}
*/
module.exports = () => {
return new Promise((resolve, reject) => {
debug('updating virus database using freshclam');
exec('freshclam', [], { stdio: 'inherit' })
.on('error', reject)
.on('exit', code => {
if (code !== 0) return void reject(new Error(`freshclam exited with code ${code}`));
debug('finished updating virus database');
resolve();
});
});
};
module.exports = function getFile (S3, key) {
return new Promise((resolve, reject) => {
S3.getObject({
Bucket: `${process.env.SERVICE}-filestore-${process.env.STAGE}-1`,
Key: key
}, (err, file) => {
if (err) return void reject(err);
resolve(file);
});
});
};
const exec = require('child_process').exec;
const debug = require('debug')('scanner');
module.exports = function refresh () {
return new Promise((resolve, reject) => {
debug('Updating virus database');
const proc = exec('freshclam', [], {stdio: 'inherit'});
proc.on('error', reject);
proc.on('exit', code => {
if (code !== 0) return void reject(new Error(`Clamscan exited with code ${code}`));
debug('Finished updating');
resolve();
});
});
};
const clam = require('clamscan')();
const fs = require('fs');
const path = require('path');
const getFile = require('./getfile');
module.exports = function scanFile (notif, S3) {
return new Promise((resolve, reject) => {
const key = notif.Records[0].s3.object.key;
getFile(S3, key).then(file => {
const filepath = path.resolve(path.join(__dirname, '/files/', key));
fs.writeFile(filepath, file.body, (err) => {
if (err) return void reject(err);
clam.is_infected(filepath, (err, _, isInfected) => {
fs.unlink(filepath, (unlinkErr) => {
if (err || unlinkErr) return void reject(err || unlinkErr);
resolve({infected: isInfected});
});
});
});
}, reject);
});
};
{ {
"name": "whats-a-virus", "name": "s3-scanner",
"version": "0.0.1", "version": "0.0.1",
"description": "Whats this, a virus? Scanner built for whats-th.is file uploader.", "description": "Node.js microservice to process events from S3 over SNS and scan new objects.",
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"test": "echo \"Error: no test specified\" && exit 1" "test": "echo \"Error: no test specified\" && exit 1"
...@@ -14,22 +14,29 @@ ...@@ -14,22 +14,29 @@
"av", "av",
"antivirus", "antivirus",
"clamscan", "clamscan",
"clamav" "clamav",
"s3",
"simple storage service",
"simple-storage-service",
"aws",
"amazon web services",
"amazon-web-services"
], ],
"author": "aurieh", "author": "Aurieh",
"license": "MIT", "license": "MIT",
"bugs": { "bugs": {
"url": "https://github.com/whats-this/scanner/issues" "url": "https://github.com/whats-this/scanner/issues"
}, },
"homepage": "https://github.com/whats-this/scanner#readme", "homepage": "https://github.com/whats-this/scanner#readme",
"dependencies": {
"aws-sdk": "^2.7.15",
"clamscan": "^0.8.4",
"debug": "^2.4.4"
},
"devDependencies": { "devDependencies": {
"eslint": "^3.12.1", "eslint": "^3.12.1",
"eslint-config-standard": "^6.2.1", "eslint-config-standard": "^6.2.1",
"eslint-plugin-promise": "^3.4.0", "eslint-plugin-promise": "^3.4.0",
"eslint-plugin-standard": "^2.0.1" "eslint-plugin-standard": "^2.0.1"
},
"dependencies": {
"clamscan": "^0.8.4",
"debug": "^2.4.4"
} }
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment