diff --git a/feature-utils/poly-import/src/index.js b/feature-utils/poly-import/src/index.js index 708b3a24f9..021da67055 100644 --- a/feature-utils/poly-import/src/index.js +++ b/feature-utils/poly-import/src/index.js @@ -14,7 +14,16 @@ import { import DataAccount from "./entities/data-account"; import { Status, statusTypes } from "../utils/status"; import { Telemetry } from "../utils/performance-telemetry"; +import { + MINIMUM_FILE_SIZE, + ZipFileEntryMock, + ZipFileMock, +} from "../utils/zipfile-mock"; +import { jsonStringifyWithUtfEscape } from "../utils/json-encoding"; + export { + MINIMUM_FILE_SIZE, + jsonStringifyWithUtfEscape, Status, Importer, runImporter, @@ -22,8 +31,10 @@ export { runOutdatedImporter, runOutdatedImporters, ZipFile, + ZipFileMock, Telemetry, ZipFileEntry, + ZipFileEntryMock, FileImportError, RefreshFilesError, FileSelectionError, diff --git a/features/facebookImport/test/utf8-encoding-decoding.test.js b/feature-utils/poly-import/test/utf8-encoding-decoding.test.js similarity index 84% rename from features/facebookImport/test/utf8-encoding-decoding.test.js rename to feature-utils/poly-import/test/utf8-encoding-decoding.test.js index 4867c01ac8..7abf1c67ff 100644 --- a/features/facebookImport/test/utf8-encoding-decoding.test.js +++ b/feature-utils/poly-import/test/utf8-encoding-decoding.test.js @@ -1,6 +1,6 @@ "use strict"; -import { jsonStringifyWithUtfEscape } from "../src/model/importers/utils/json-encoding"; +import { jsonStringifyWithUtfEscape } from "../utils/json-encoding"; const jsonData1 = "🦊"; const jsonData2 = "🦊🤗😍"; @@ -22,6 +22,11 @@ describe("JSON encode", () => { it("Various characters", () => { expect(jsonStringifyWithUtfEscape(jsonData3)).toBe(decodedData3); }); + + it("passes through non-string data", () => { + const foo42 = { foo: 42 }; + expect(jsonStringifyWithUtfEscape(foo42)).toBe(JSON.stringify(foo42)); + }); }); describe("Decodes", () => { diff --git a/features/facebookImport/src/model/importers/utils/json-encoding.js b/feature-utils/poly-import/utils/json-encoding.js similarity index 91% rename from features/facebookImport/src/model/importers/utils/json-encoding.js rename to feature-utils/poly-import/utils/json-encoding.js index e62b4544ca..db1e5b17dc 100644 --- a/features/facebookImport/src/model/importers/utils/json-encoding.js +++ b/feature-utils/poly-import/utils/json-encoding.js @@ -24,8 +24,7 @@ function jsonStringifyWithUtfEscape(jsonData) { } return value; }); - const escapedString = jsonStringEscapeUtfCharacters(serializedString); - return escapedString; + return jsonStringEscapeUtfCharacters(serializedString); } export { jsonStringEscapeUtfCharacters, jsonStringifyWithUtfEscape }; diff --git a/features/facebookImport/test/mocks/zipfile-mock.js b/feature-utils/poly-import/utils/zipfile-mock.js similarity index 96% rename from features/facebookImport/test/mocks/zipfile-mock.js rename to feature-utils/poly-import/utils/zipfile-mock.js index 024cd84cf7..6341c2628b 100644 --- a/features/facebookImport/test/mocks/zipfile-mock.js +++ b/feature-utils/poly-import/utils/zipfile-mock.js @@ -1,4 +1,4 @@ -import { jsonStringifyWithUtfEscape } from "../../src/model/importers/utils/json-encoding"; +import { jsonStringifyWithUtfEscape } from "./json-encoding"; //The minimum size of a .ZIP file is 22 bytes export const MINIMUM_FILE_SIZE = 22; diff --git a/features/facebookImport/test/datasets/messages-data.js b/features/facebookImport/test/datasets/messages-data.js index 09ad46ac59..c49ea1a4fa 100644 --- a/features/facebookImport/test/datasets/messages-data.js +++ b/features/facebookImport/test/datasets/messages-data.js @@ -1,4 +1,4 @@ -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { createMockedZip } from "../utils/data-creation"; export const DATASET_EXPECTED_VALUES = { numberOfMessageThreads: 2, diff --git a/features/facebookImport/test/datasets/posts-data.js b/features/facebookImport/test/datasets/posts-data.js index 8d5f40f1df..358193fae1 100644 --- a/features/facebookImport/test/datasets/posts-data.js +++ b/features/facebookImport/test/datasets/posts-data.js @@ -1,4 +1,4 @@ -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { createMockedZip } from "../utils/data-creation"; const POSTS_FILE_PATH = "posts/your_posts_X.json"; diff --git a/features/facebookImport/test/importers/ad-interests-importer.test.js b/features/facebookImport/test/importers/ad-interests-importer.test.js index d1383c2a88..395b376bff 100644 --- a/features/facebookImport/test/importers/ad-interests-importer.test.js +++ b/features/facebookImport/test/importers/ad-interests-importer.test.js @@ -5,7 +5,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithAdInterests, } from "../datasets/ad-interests-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runAdInterestsImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/ad-views-importer.test.js b/features/facebookImport/test/importers/ad-views-importer.test.js index cbb57ac966..62d6a429b2 100644 --- a/features/facebookImport/test/importers/ad-views-importer.test.js +++ b/features/facebookImport/test/importers/ad-views-importer.test.js @@ -9,7 +9,7 @@ import { createEnglishDatasetWithMissingAdsCategory, createIncompleteEnglishAdViewsData, } from "../datasets/ad-views-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runAdsImportForDataset, diff --git a/features/facebookImport/test/importers/comments-importer.test.js b/features/facebookImport/test/importers/comments-importer.test.js index d2452d45ca..a505b7bfd5 100644 --- a/features/facebookImport/test/importers/comments-importer.test.js +++ b/features/facebookImport/test/importers/comments-importer.test.js @@ -5,7 +5,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithComments, } from "../datasets/comments-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runCommentsImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/connected-advertisers-all-types-importer.test.js b/features/facebookImport/test/importers/connected-advertisers-all-types-importer.test.js index 4e34a32119..bcf4125b83 100644 --- a/features/facebookImport/test/importers/connected-advertisers-all-types-importer.test.js +++ b/features/facebookImport/test/importers/connected-advertisers-all-types-importer.test.js @@ -6,7 +6,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithConnectedAdvertisersAllTypes, } from "../datasets/connected-advertisers-all-types-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runConnectedAdvertisersWithAllTypesImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/connected-advertisers-importer.test.js b/features/facebookImport/test/importers/connected-advertisers-importer.test.js index 3f6c623634..8529ce1ea7 100644 --- a/features/facebookImport/test/importers/connected-advertisers-importer.test.js +++ b/features/facebookImport/test/importers/connected-advertisers-importer.test.js @@ -6,7 +6,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithConnectedAdvertisers, } from "../datasets/connected-advertisers-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runConnectedAdvertisersImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/interacted-with-advertisers-importer.test.js b/features/facebookImport/test/importers/interacted-with-advertisers-importer.test.js index 9654486f5e..a624ee3e97 100644 --- a/features/facebookImport/test/importers/interacted-with-advertisers-importer.test.js +++ b/features/facebookImport/test/importers/interacted-with-advertisers-importer.test.js @@ -5,7 +5,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithInteractedWithAdvertisers, } from "../datasets/interacted-with-advertisers-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runInteractedWithAdvertisersImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/language-and-locale-importer.test.js b/features/facebookImport/test/importers/language-and-locale-importer.test.js index a36bef40b5..5a09b4ddfc 100644 --- a/features/facebookImport/test/importers/language-and-locale-importer.test.js +++ b/features/facebookImport/test/importers/language-and-locale-importer.test.js @@ -5,7 +5,7 @@ import { createLanguageSettingsData, createLocaleData, } from "../datasets/language-and-locale-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runLanguageAndLocaleImporter } from "../utils/data-importing"; import { expectImportSuccess, diff --git a/features/facebookImport/test/importers/messages-importer.test.js b/features/facebookImport/test/importers/messages-importer.test.js index a0082e6a2b..4121b441c7 100644 --- a/features/facebookImport/test/importers/messages-importer.test.js +++ b/features/facebookImport/test/importers/messages-importer.test.js @@ -10,7 +10,7 @@ import { zipFileWithMessageThreads, zipFileWithThreeFileErrors, } from "../datasets/messages-data.js"; -import { ZipFileMock } from "../mocks/zipfile-mock.js"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runMessagesImporter } from "../utils/data-importing.js"; import { expectError, diff --git a/features/facebookImport/test/importers/name-importer.test.js b/features/facebookImport/test/importers/name-importer.test.js index f873bfcd30..1bda32a7ff 100644 --- a/features/facebookImport/test/importers/name-importer.test.js +++ b/features/facebookImport/test/importers/name-importer.test.js @@ -1,7 +1,7 @@ "use strict"; import NameImporter from "../../src/model/importers/name-importer"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runNameImporter } from "../utils/data-importing"; import { expectError, diff --git a/features/facebookImport/test/importers/off-facebook-events-importer.test.js b/features/facebookImport/test/importers/off-facebook-events-importer.test.js index eff7cac237..a7fd27f325 100644 --- a/features/facebookImport/test/importers/off-facebook-events-importer.test.js +++ b/features/facebookImport/test/importers/off-facebook-events-importer.test.js @@ -7,7 +7,7 @@ import { zipFileWithOffFacebookEvents, DATASET_EXPECTED_VALUES, } from "../datasets/off-facebook-events-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runOffFacebookEventsImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/personal-data-importer.test.js b/features/facebookImport/test/importers/personal-data-importer.test.js index 5189d621cd..503cd27db5 100644 --- a/features/facebookImport/test/importers/personal-data-importer.test.js +++ b/features/facebookImport/test/importers/personal-data-importer.test.js @@ -1,7 +1,7 @@ import PersonalDataImporter, { PROFILE_INFORMATION_FILE_PATH, } from "../../src/model/importers/personal-data-importer"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runPersonalDataImporter } from "../utils/data-importing"; import { expectError, diff --git a/features/facebookImport/test/importers/post-reactions-importer.test.js b/features/facebookImport/test/importers/post-reactions-importer.test.js index 46dd0d871f..7b3a95076a 100644 --- a/features/facebookImport/test/importers/post-reactions-importer.test.js +++ b/features/facebookImport/test/importers/post-reactions-importer.test.js @@ -5,7 +5,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithPostReactions, } from "../datasets/post-reactions-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runPostReactionsImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/posts-importer.test.js b/features/facebookImport/test/importers/posts-importer.test.js index b98215b0b7..3662438f40 100644 --- a/features/facebookImport/test/importers/posts-importer.test.js +++ b/features/facebookImport/test/importers/posts-importer.test.js @@ -11,7 +11,7 @@ import { zipFileWithTwoFileErrors, zipFileWithTwoPostsFiles, } from "../datasets/posts-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runPostsImporter } from "../utils/data-importing"; import { expectError, diff --git a/features/facebookImport/test/importers/searches-importer.test.js b/features/facebookImport/test/importers/searches-importer.test.js index cb0485b994..f134ac1e27 100644 --- a/features/facebookImport/test/importers/searches-importer.test.js +++ b/features/facebookImport/test/importers/searches-importer.test.js @@ -5,7 +5,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithSearches, } from "../datasets/searches-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../utils/data-creation"; import { runSearchesImporter } from "../utils/data-importing"; import { diff --git a/features/facebookImport/test/importers/test-definition/importer-tests-definition.js b/features/facebookImport/test/importers/test-definition/importer-tests-definition.js index dc1678f45d..ba224d5c96 100644 --- a/features/facebookImport/test/importers/test-definition/importer-tests-definition.js +++ b/features/facebookImport/test/importers/test-definition/importer-tests-definition.js @@ -1,4 +1,4 @@ -import { ZipFileMock } from "../../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { zipWithWrongDatasetKey } from "../../utils/data-creation"; import { runSingleOutdatedImporter } from "../../utils/data-importing"; import { diff --git a/features/facebookImport/test/ministory/about-pictures-data-analysis.test.js b/features/facebookImport/test/ministory/about-pictures-data-analysis.test.js index cb766a4df0..d702df9683 100644 --- a/features/facebookImport/test/ministory/about-pictures-data-analysis.test.js +++ b/features/facebookImport/test/ministory/about-pictures-data-analysis.test.js @@ -1,6 +1,6 @@ import AboutPicturesDataAnalysis from "../../src/model/analyses/ministories/about-pictures-data-analysis"; import AboutPicturesMinistory from "../../src/views/ministories/aboutPictures"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runAnalysisForExport } from "../utils/analyses-execution"; import { expectInactiveAnalysis, diff --git a/features/facebookImport/test/ministory/json-file-names-analysis.test.js b/features/facebookImport/test/ministory/json-file-names-analysis.test.js new file mode 100644 index 0000000000..e738751cad --- /dev/null +++ b/features/facebookImport/test/ministory/json-file-names-analysis.test.js @@ -0,0 +1,72 @@ +import JSONFileNamesAnalysis from "../../src/model/analyses/report/json-file-names-analysis"; +import commonStructure from "../../src/static/commonStructure"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; +import { runAnalysisForExport } from "../utils/analyses-execution"; +import { expectAnalysisSuccessStatus } from "../utils/analysis-assertions"; + +const commonJsonFiles = commonStructure + .filter((path) => path.match(/\.json$/)) + .map((jsonPath) => { + return jsonPath.substring(1); + }); + +async function analyzeZipWithFiles(files) { + const zipFile = new ZipFileMock(); + if (files.length > 0) { + files.forEach((jsonPath) => { + zipFile.addJsonEntry(jsonPath, { foo: "bar" }); + }); + } + const { analysisResult } = await runAnalysisForExport( + JSONFileNamesAnalysis, + zipFile + ); + return analysisResult; +} + +describe("JSON files analysis for empty zip", () => { + let status; + beforeAll(async () => { + ({ status } = await analyzeZipWithFiles([])); + }); + it("has success status", async () => { + expectAnalysisSuccessStatus(status); + }); +}); + +describe("JSON files analysis for non-empty zip", () => { + let status; + let analysis; + + beforeAll(async () => { + ({ status, analysis } = await analyzeZipWithFiles(commonJsonFiles)); + }); + + it("reports successful status", () => { + expect(status.isSuccess).toBe(true); + }); + + it("has the right name", () => { + expect(analysis.reportData).toStrictEqual(commonJsonFiles); + }); + + it("has the right title", () => { + expect(analysis.title).toBe("JSON file names"); + }); +}); + +describe("JSON files analysis with anonymized path", () => { + let analysis; + beforeAll(async () => { + commonJsonFiles.push( + "messages/archived_threads/facebookuser_gktxomrifg/message_1.json" + ); + ({ analysis } = await analyzeZipWithFiles(commonJsonFiles)); + }); + + it("has anonymized file", () => { + expect(analysis.reportData).toContain( + "messages/archived_threads/uniqueid_hash/message_1.json" + ); + }); +}); diff --git a/features/facebookImport/test/ministory/messages-analysis.test.js b/features/facebookImport/test/ministory/messages-analysis.test.js index 6668d836af..cd4449fde1 100644 --- a/features/facebookImport/test/ministory/messages-analysis.test.js +++ b/features/facebookImport/test/ministory/messages-analysis.test.js @@ -4,7 +4,7 @@ import { DATASET_EXPECTED_VALUES, zipFileWithMessageThreads, } from "../datasets/messages-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runAnalysisForExport } from "../utils/analyses-execution"; import { expectActiveAnalysis, diff --git a/features/facebookImport/test/ministory/missing-common-json-files.test.js b/features/facebookImport/test/ministory/missing-common-json-files.test.js index 0abc9d6a9d..5e4ce8fc69 100644 --- a/features/facebookImport/test/ministory/missing-common-json-files.test.js +++ b/features/facebookImport/test/ministory/missing-common-json-files.test.js @@ -1,7 +1,7 @@ import MissingCommonJSONFilesAnalysis from "../../src/model/analyses/report/missing-common-json-files"; import commonStructure from "../../src/static/commonStructure"; import MissingCommonJSONFilesReport from "../../src/views/ministories/missingCommonJsonFiles"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runAnalysisForExport } from "../utils/analyses-execution"; import { expectActiveAnalysis, diff --git a/features/facebookImport/test/ministory/post-reactions-types-analysis.test.js b/features/facebookImport/test/ministory/post-reactions-types-analysis.test.js index 1862f4ca91..d701f07c5f 100644 --- a/features/facebookImport/test/ministory/post-reactions-types-analysis.test.js +++ b/features/facebookImport/test/ministory/post-reactions-types-analysis.test.js @@ -1,7 +1,7 @@ import PostReactionsTypesAnalysis from "../../src/model/analyses/ministories/post-reactions-types-analysis"; import PostReactionTypesMinistory from "../../src/views/ministories/postReactionsTypes"; import { zipFileWithPostReactions } from "../datasets/post-reactions-data"; -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { runAnalysisForExport } from "../utils/analyses-execution"; import { expectActiveAnalysis, diff --git a/features/facebookImport/test/ministory/report-metadata.test.js b/features/facebookImport/test/ministory/report-metadata.test.js index 35d56e0377..6f56887f05 100644 --- a/features/facebookImport/test/ministory/report-metadata.test.js +++ b/features/facebookImport/test/ministory/report-metadata.test.js @@ -7,7 +7,7 @@ import ReportMetadataReport from "../../src/views/ministories/reportMetadata"; import { createInteractedWithAdvertisersDataset } from "../datasets/interacted-with-advertisers-data"; import { createLanguageSettingsData } from "../datasets/language-and-locale-data"; import { createOffFacebookEventsSimpleData } from "../datasets/off-facebook-events-data"; -import { MINIMUM_FILE_SIZE } from "../mocks/zipfile-mock"; +import { MINIMUM_FILE_SIZE } from "@polypoly-eu/poly-import"; import { runAnalysisForExport } from "../utils/analyses-execution"; import { expectActiveAnalysis, diff --git a/features/facebookImport/test/utils/data-creation.js b/features/facebookImport/test/utils/data-creation.js index 605a9b6da0..df4b12e9c6 100644 --- a/features/facebookImport/test/utils/data-creation.js +++ b/features/facebookImport/test/utils/data-creation.js @@ -1,4 +1,4 @@ -import { ZipFileMock } from "../mocks/zipfile-mock"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; export function createMockedZip(datasets) { let zipFile = new ZipFileMock(); diff --git a/features/facebookImport/test/utils/data-importing.js b/features/facebookImport/test/utils/data-importing.js index e49ab20e91..b3391c32bc 100644 --- a/features/facebookImport/test/utils/data-importing.js +++ b/features/facebookImport/test/utils/data-importing.js @@ -12,7 +12,7 @@ import RecentlyViewedAdsImporter, { RECENTLY_VIEWED_FILE_PATH, } from "../../src/model/importers/recently-viewed-ads-importer.js"; import OffFacebookEventsImporter from "../../src/model/importers/off-facebook-events-importer.js"; -import { ZipFileMock } from "../mocks/zipfile-mock.js"; +import { ZipFileMock } from "@polypoly-eu/poly-import"; import { MockPod } from "@polypoly-eu/api/dist/mock-pod"; import LanguageAndLocaleImporter from "../../src/model/importers/language-and-locale-importer.js"; import FriendsImporter from "../../src/model/importers/friends-importer.js";