Refactored loader and added tests

This commit is contained in:
nsieg 2022-11-19 13:38:44 +01:00
parent 3994a1ca63
commit e5cf254d9f
9 changed files with 324 additions and 155 deletions

View file

@ -1,10 +1,9 @@
const fsprom = require("fs").promises;
const path = require("path");
const fs = require("fs");
const superagent = require("superagent");
async function getImages(rootPath) {
const content = await fsprom.readdir(rootPath, { withFileTypes: true });
const content = await fs.promises.readdir(rootPath, { withFileTypes: true });
const images = content
.filter((x) => !x.isDirectory())
.map((x) => x.name)
@ -14,20 +13,12 @@ async function getImages(rootPath) {
async function deleteImages(rootPath, fileNamesLocal, fileNamesServer) {
const diff = fileNamesLocal.filter((x) => !fileNamesServer.includes(x));
try {
for (fileName of diff) {
//console.log("delete " + path.join("/", rootPath, fileName));
await fsprom.unlink(path.join("/", rootPath, fileName));
}
} catch (err) {
// do not propagate errors so that failed delete will not lead to failed download
console.log(err);
for (fileName of diff) {
await fs.promises.unlink(path.join("/", rootPath, fileName));
}
}
async function downloadImages(rootPath, config) {
const localFileNames = await getImages(rootPath);
const agent = superagent.agent();
async function fetchServerImages(agent, config) {
await agent
.post(config.url + "?format=json&method=pwg.session.login")
.field("username", config.username)
@ -37,22 +28,39 @@ async function downloadImages(rootPath, config) {
);
respText = JSON.parse(data.res.text);
images = respText.result.images;
serverFileNames = [];
for (img of images) {
const fileName = img.file;
serverFileNames.push(fileName);
// Only download if file does not yet exist
if (!localFileNames.some((x) => x === fileName)) {
await new Promise((resolve) =>
agent
.get(img.derivatives[config.size].url)
.pipe(fs.createWriteStream(path.join("/", rootPath, fileName)))
.on("finish", resolve)
);
}
}
await deleteImages(rootPath, localFileNames, serverFileNames);
if (images.length === 0) {
throw Error(
"There were no images found! Check connection details or upload images."
);
}
return images;
}
module.exports = { getImages, downloadImages };
async function downloadImage(agent, url, targetPath) {
await new Promise((resolve) =>
agent.get(url).pipe(fs.createWriteStream(targetPath)).on("finish", resolve)
);
}
async function syncImages(rootPath, config) {
const agent = superagent.agent();
const images = await fetchServerImages(agent, config);
const localFileNames = await getImages(rootPath);
const newImages = images.filter((img) => !localFileNames.includes(img.file));
for (img of newImages) {
const url = img.derivatives[config.size].url;
await downloadImage(agent, url, path.join("/", rootPath, img.file));
}
await deleteImages(
rootPath,
localFileNames,
images.map((i) => i.file)
);
}
const exportedForTesting = { deleteImages, fetchServerImages, downloadImage };
module.exports = { getImages, syncImages, exportedForTesting };

View file

@ -1,7 +1,7 @@
const NodeHelper = require("node_helper");
const path = require("path");
const _ = require("lodash");
const { getImages, downloadImages } = require("./loader");
const { getImages, syncImages } = require("./loader");
const Log = require("logger");
module.exports = NodeHelper.create({
@ -11,7 +11,7 @@ module.exports = NodeHelper.create({
syncImages: function (config) {
const imagePath = path.join("/", __dirname, "images");
downloadImages(imagePath, config)
syncImages(imagePath, config)
.then(() => {
getImages(imagePath).then((images) => {
shuffledImages = _.shuffle(images);
@ -19,7 +19,7 @@ module.exports = NodeHelper.create({
});
})
.catch((err) => {
Log.error("Could not download images from piwigo! Error was: ");
Log.error("[piwigo] Error: ");
Log.error(err);
});
},

51
package-lock.json generated
View file

@ -8,13 +8,14 @@
"name": "mmm-piwigo-photo-wheel",
"version": "1.0.0",
"dependencies": {
"chai": "^4.3.6",
"mocha": "^10.1.0",
"superagent": "^8.0.3",
"superagent-mock": "^4.0.0"
"superagent": "^8.0.3"
},
"devDependencies": {
"prettier": "2.7.1"
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^10.1.0",
"prettier": "2.7.1",
"superagent-mock": "^4.0.0"
}
},
"node_modules/ansi-colors": {
@ -159,6 +160,18 @@
"node": ">=4"
}
},
"node_modules/chai-as-promised": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
"integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
"dev": true,
"dependencies": {
"check-error": "^1.0.2"
},
"peerDependencies": {
"chai": ">= 2.1.2 < 5"
}
},
"node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@ -445,19 +458,6 @@
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
},
"node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@ -1323,6 +1323,15 @@
"type-detect": "^4.0.5"
}
},
"chai-as-promised": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
"integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
"dev": true,
"requires": {
"check-error": "^1.0.2"
}
},
"chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@ -1530,12 +1539,6 @@
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
},
"fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"optional": true
},
"function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",

View file

@ -7,12 +7,13 @@
"start": "node test_manual/serve.js"
},
"dependencies": {
"chai": "^4.3.6",
"mocha": "^10.1.0",
"superagent": "^8.0.3",
"superagent-mock": "^4.0.0"
"superagent": "^8.0.3"
},
"devDependencies": {
"prettier": "2.7.1"
"chai-as-promised": "^7.1.1",
"prettier": "2.7.1",
"superagent-mock": "^4.0.0",
"chai": "^4.3.6",
"mocha": "^10.1.0"
}
}

12
test/response-empty.json Normal file
View file

@ -0,0 +1,12 @@
{
"stat": "ok",
"result": {
"paging": {
"page": 0,
"per_page": 100,
"count": 1,
"total_count": "1"
},
"images": []
}
}

81
test/response.json Normal file
View file

@ -0,0 +1,81 @@
{
"stat": "ok",
"result": {
"paging": {
"page": 0,
"per_page": 100,
"count": 1,
"total_count": "1"
},
"images": [
{
"is_favorite": false,
"id": 1,
"width": 4016,
"height": 5197,
"hit": 0,
"file": "demo-image.jpg",
"name": "demo-image",
"comment": null,
"date_creation": null,
"date_available": "2022-11-01 21:14:08",
"page_url": "https://piwigo.example/picture.php?/1",
"element_url": "https://piwigo.example/upload/2022/11/01/20221101211408-9a0c06b1.jpg",
"derivatives": {
"square": {
"url": "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-sq.jpg",
"width": 120,
"height": 120
},
"thumb": {
"url": "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-th.jpg",
"width": 111,
"height": 144
},
"2small": {
"url": "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-2s.jpg",
"width": 185,
"height": 240
},
"xsmall": {
"url": "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-xs.jpg",
"width": 250,
"height": 324
},
"small": {
"url": "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-sm.jpg",
"width": 333,
"height": 432
},
"medium": {
"url": "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-me.jpg",
"width": 459,
"height": 594
},
"large": {
"url": "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-la.jpg",
"width": 584,
"height": 756
},
"xlarge": {
"url": "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-xl.jpg",
"width": 709,
"height": 918
},
"xxlarge": {
"url": "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-xx.jpg",
"width": 959,
"height": 1242
}
},
"categories": [
{
"id": 1,
"url": "https://piwigo.example/index.php?/category/1",
"page_url": "https://piwigo.example/picture.php?/1/category/1"
}
]
}
]
}
}

View file

@ -0,0 +1,36 @@
const response = require("./response-empty.json");
module.exports = [
{
pattern: "https://piwigo.example(.*)",
fixtures: function (match, params, headers, context) {
if (match[1] === "?format=json&method=pwg.session.login") {
return {
stat: "ok",
result: true,
};
}
if (match[1] === "?format=json&method=pwg.categories.getImages") {
return JSON.stringify(response);
}
},
get: function (match, data) {
return {
status: 200,
res: {
text: data,
},
};
},
post: function (match, data) {
return {
status: 200,
res: {
text: data,
},
};
},
},
];

View file

@ -1,3 +1,5 @@
const response = require("./response.json");
module.exports = [
{
pattern: "https://piwigo.example(.*)",
@ -19,99 +21,23 @@ module.exports = [
}
if (match[1] === "?format=json&method=pwg.categories.getImages") {
return {
stat: "ok",
result: {
paging: {
page: 0,
per_page: 100,
count: 1,
total_count: "1",
},
images: [
{
is_favorite: false,
id: 1,
width: 4016,
height: 5197,
hit: 0,
file: "matthew-brodeur-DH_u2aV3nGM-unsplash.jpg",
name: "matthew-brodeur-DH u2aV3nGM-unsplash",
comment: null,
date_creation: null,
date_available: "2022-11-01 21:14:08",
page_url: "https://piwigo.example/picture.php?/1",
element_url:
"https://piwigo.example/upload/2022/11/01/20221101211408-9a0c06b1.jpg",
derivatives: {
square: {
url: "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-sq.jpg",
width: 120,
height: 120,
},
thumb: {
url: "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-th.jpg",
width: 111,
height: 144,
},
"2small": {
url: "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-2s.jpg",
width: 185,
height: 240,
},
xsmall: {
url: "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-xs.jpg",
width: 250,
height: 324,
},
small: {
url: "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-sm.jpg",
width: 333,
height: 432,
},
medium: {
url: "https://piwigo.example/_data/i/upload/2022/11/01/20221101211408-9a0c06b1-me.jpg",
width: 459,
height: 594,
},
large: {
url: "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-la.jpg",
width: 584,
height: 756,
},
xlarge: {
url: "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-xl.jpg",
width: 709,
height: 918,
},
xxlarge: {
url: "https://piwigo.example/i.php?/upload/2022/11/01/20221101211408-9a0c06b1-xx.jpg",
width: 959,
height: 1242,
},
},
categories: [
{
id: 1,
url: "https://piwigo.example/index.php?/category/1",
page_url:
"https://piwigo.example/picture.php?/1/category/1",
},
],
},
],
},
};
return JSON.stringify(response);
}
},
get: function (match, data) {
return {
body: data,
status: 200,
res: {
text: data,
},
};
},
post: function (match, data) {
return {
status: 201,
status: 200,
res: {
text: data,
},
};
},
},

View file

@ -1,15 +1,21 @@
const path = require("path");
const fs = require("fs").promises;
const os = require("os");
const chai = require("chai");
const chaiAsPromised = require("chai-as-promised");
chai.use(chaiAsPromised);
const expect = chai.expect;
const request = require("superagent");
const mockConfig = require("./superagent-mock-config");
const mockConfigEmpty = require("./superagent-mock-config-empty");
const { getImages, downloadImages } = require("./../loader");
const { getImages, syncImages, exportedForTesting } = require("./../loader");
const { deleteImages, fetchServerImages, downloadImage } = exportedForTesting;
describe("Loader", function () {
describe("getImages()", function () {
describe("getImages", function () {
it("should return array of file names", async () => {
// Given
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "foo-"));
@ -19,6 +25,7 @@ describe("Loader", function () {
// Then
expect(data).to.eql(["mynewfile1.txt"]);
});
it("should ignore .gitkeep", async () => {
// Given
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "foo-"));
@ -31,20 +38,115 @@ describe("Loader", function () {
expect(data).to.eql([".gitbla", "mynewfile1.txt"]);
});
});
});
describe("Loader", function () {
describe("downloadImages()", function () {
it("should connect", async () => {
describe("deleteImages", function () {
it("should delete local-only files", async () => {
// Given
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "foo-"));
await fs.appendFile(dir + "/local.txt", "I exist only locally");
await fs.appendFile(dir + "/both.txt", "I exist locally and remotely");
const localFileNames = ["local.txt"];
const serverFileNames = ["server.txt", "both.txt"];
// When
await deleteImages(dir, localFileNames, serverFileNames);
// Then
const content = await fs.readdir(dir, { withFileTypes: true });
expect(content.map((x) => x.name)).not.to.include("local.txt");
expect(content.map((x) => x.name)).to.include("both.txt");
});
it("should throw on deletion error", async () => {
// Given
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "foo-"));
await fs.appendFile(dir + "/else.txt", "hello");
const lfn = ["local.txt"];
const sfn = ["server.txt"];
// When Then
return expect(deleteImages(dir, lfn, sfn))
.to.be.eventually.rejectedWith(
"ENOENT: no such file or directory, unlink '" + dir + "/local.txt'"
)
.and.be.an.instanceOf(Error);
});
});
describe("fetchServerImages", function () {
before(() => {
this.superagentMock = require("superagent-mock")(request, mockConfig);
});
after(() => {
this.superagentMock.unset();
});
it("should fetch and parse correctly", async () => {
// Given
const config = {
username: "tester",
password: "secret",
url: "https://piwigo.example",
size: "large",
};
var superagentMock = require("superagent-mock")(request, mockConfig);
await downloadImages(path.join("/", __dirname, "../images"), config);
superagentMock.unset();
// When
const images = await fetchServerImages(request.agent(), config);
// Then
expect(images).to.have.length(1);
expect(images[0].file).to.eql("demo-image.jpg");
});
it("should throw on connection error", async () => {
// Given
const config = {
username: "tester",
password: "secret",
url: "https://piwigo.wrong",
size: "large",
};
// When Then
return expect(fetchServerImages(request.agent(), config))
.to.be.eventually.rejectedWith("ENOTFOUND piwigo.wrong")
.and.be.an.instanceOf(Error);
});
});
describe("fetchServerImages - Empty", function () {
before(() => {
this.superagentMock = require("superagent-mock")(
request,
mockConfigEmpty
);
});
after(() => {
this.superagentMock.unset();
});
it("should throw on empty list", async () => {
// Given
const config = {
username: "tester",
password: "secret",
url: "https://piwigo.example",
size: "large",
};
// When Then
return expect(fetchServerImages(request.agent(), config))
.to.be.eventually.rejectedWith(
"There were no images found! Check connection details or upload images"
)
.and.be.an.instanceOf(Error);
});
});
describe("syncImages", function () {
xit("should connect (manual integration-test)", async () => {
const config = {
username: "<real-user>",
password: "<real-password>",
url: "<real-url>",
size: "large",
};
await syncImages(path.join("/", __dirname, "../images"), config);
}).timeout(30000);
});
});