🚧 back: wip auto-create subdirectories as required when creating Driv…

…eFile (#714)

tdrive/backend/node/src/services/documents/services/index.ts

@@ -370,10 +370,73 @@ export class DocumentsService {
     context: DriveExecutionContext,
   ): Promise<DriveFile> => {
     try {
-      const driveItem = getDefaultDriveItem(content, context);
-      const driveItemVersion = getDefaultDriveItemVersion(version, context);
+      const driveItem = getDefaultDriveItem(content, context) as DriveFile; //TODO Why is this cast needed
+      const driveItemVersion = getDefaultDriveItemVersion(version, context) as FileVersion; //TODO Why is this cast needed
       driveItem.scope = await getItemScope(driveItem, this.repository, context);
 
+      const createdFolderIds = [];
+      const pathComponents = driveItem.name.split("/"); //TODO Better path management
+      if (pathComponents[0] === "") pathComponents.shift(); // tolerate prefix /
+      if (
+        driveItem.is_directory &&
+        pathComponents.length &&
+        pathComponents[pathComponents.length - 1] === ""
+      )
+        pathComponents.pop(); // tolerate suffix / but only for directories
+      if (pathComponents.length === 0 || pathComponents.some(x => !x || x === "." || x === ".."))
+        throw new Error(`Invalid path: ${JSON.stringify(driveItem.name)}`);
+      if (pathComponents.length > 1) {
+        driveItem.name = pathComponents.pop();
+        if (!driveItem.name?.length)
+          throw new Error(`Invalid path ${JSON.stringify(driveItem.name)}: cannot end with a /`);
+        let lastParentId = driveItem.parent_id;
+        let couldNextExist = true;
+        logger.info(
+          { path: driveItem.name, pathComponents, parentId: lastParentId },
+          "Creating intermediary folders",
+        );
+        for (const folderName of pathComponents) {
+          const existing =
+            couldNextExist &&
+            (await this.repository.findOne(
+              { company_id: context.company.id, parent_id: lastParentId, name: folderName },
+              {},
+              context,
+            ));
+          if (existing) {
+            logger.debug({ folderName, id: existing.id }, " !!!! Already exists"); //TODO NONONO
+            if (existing.is_in_trash)
+              throw new Error(
+                `Error creating intermediary path ${JSON.stringify(
+                  folderName,
+                )} under ${JSON.stringify(driveItem.parent_id)}: ${JSON.stringify(
+                  folderName,
+                )} is in the trash`,
+              );
+            lastParentId = existing.id;
+            continue;
+          }
+          const newFolder = getDefaultDriveItem(
+            {
+              parent_id: lastParentId,
+              name: folderName,
+              is_directory: true,
+            },
+            context,
+          ) as DriveFile; //TODO Why is this cast needed
+          newFolder.scope = driveItem.scope;
+          await this.repository.save(newFolder, context);
+          logger.info(
+            { folderName, id: newFolder.id, parentId: lastParentId },
+            "Auto-created folder",
+          );
+          lastParentId = newFolder.id;
+          createdFolderIds.push(newFolder.id);
+          couldNextExist = false;
+        }
+        driveItem.parent_id = lastParentId;
+      }
+
       const hasAccess = await checkAccess(
         driveItem.parent_id,
         null,

tdrive/frontend/src/app/components/file-uploads/pending-file-components/pending-upload-jobs-popup.stories.tsx

@@ -0,0 +1,93 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import React from 'react';
+import { ComponentStory } from '@storybook/react';
+// import { PendingUploadJobsPopup } from './pending-upload-jobs-popup';
+
+
+// Job
+
+
+
+import CloseIcon from '@material-ui/icons/CloseOutlined';
+
+export const PendingUploadJobsPopupHeader = (props: {
+  isListingFiles?: boolean,
+  failuresListing?: string[],
+}) => {
+  return <div className='
+    w-full
+    flex
+    flex-col
+    items-center
+    '>
+      <div className='whitespace-nowrap font-extrabold text-xl'>Uploads</div>
+      <div className='whitespace-nowrap'>Currently scanning or something</div>
+  </div>
+};
+
+export const PendingUploadJobRow = () => {
+  return <div className='
+    w-full
+    flex
+    flex-row
+    items-center
+    '>
+      <div className='whitespace-nowrap flex-shrink' style={{border: '1px solid green'}}>
+        I'm the header !
+        <br />
+        Line 2
+      </div>
+      <div style={{border: '1px solid blue'}}>
+        <CloseIcon className="h-2 w-2" />
+      </div>
+      <div style={{border: '1px solid yellow'}}>
+        {/* <CloseIcon className="m-icon" /> */}
+        <CloseIcon />yo
+      </div>
+  </div>
+};
+
+export const PendingUploadJobsPopup = () => {
+  return <>
+    <div className='
+      bg-slate-200 text-black dark:bg-slate-800 dark:text-white
+      border-blue-500 border-2 border-b-0 border-solid
+      max-w-lg absolute z-50 right-10 bottom-0 w-32
+      rounded-t-lg
+      flex
+      flex-col
+    '>
+      <PendingUploadJobsPopupHeader />
+      <div className='
+        overflow-y-scroll
+        max-h-14
+        flex-col
+        '>
+        <PendingUploadJobRow />
+        <PendingUploadJobRow />
+        <PendingUploadJobRow />
+        <PendingUploadJobRow />
+        <PendingUploadJobRow />
+      </div>
+    </div>
+  </>;
+};
+
+
+
+
+
+export default {
+  //title: '@atoms/pending-upload-jobs-popup', //TODO NONONO
+  component: PendingUploadJobsPopup,
+};
+
+
+const Template: ComponentStory<typeof PendingUploadJobsPopup> = PendingUploadJobsPopup;
+
+export const Base = Template.bind({});
+// Base.play = () => {
+//   // setTimeout(() => window.alert("hi"), 1000)
+// }
+Base.args = {
+};

tdrive/frontend/src/app/components/uploads/file-tree-utils.ts

@@ -1,15 +1,46 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
+import { sortNaturally } from "src/utils/array";
+
+/** @deprecated */
 type TreeItem = { [key: string]: File | TreeItem };
 
+/** @deprecated use {@see UploadJobRoot} from now on */
 export type FileTreeObject = {
   tree: TreeItem;
   documentsCount: number;
   totalSize: number;
 };
 
+type UploadItemCommon = { error?: DOMException; };
+type UploadFileItemSpecific = { isFile: true;  entry: FileSystemFileEntry; getFile: () => Promise<File> };
+type UploadEmptyDirectoryItemSpecific = { isFile: false; entry: FileSystemDirectoryEntry; };
+
+/** Represents a single item to process, either a file to upload, or an empty folder to create, at `.entry.fullPath` */
+export type UploadItem = UploadItemCommon & (UploadFileItemSpecific | UploadEmptyDirectoryItemSpecific);
+
+/**
+ * When dropping multiple items at once, each is a `UploadJobRoot`, this also contains a flat list of all descendant items.
+ * A Job/Root in this sense is an entry, the progression of which is visible to the user. It corresponds to each separate
+ * thing the user selected to drop. `titleItem` is just for display, the same item is included in `items` if it requires
+ * an upload.
+ */
+type UploadJobRoot = {
+  titleItem: UploadItem;
+  items: UploadItem[];
+};
+
+/**
+ * Result of enumerating a single drop event.
+ * It may have multiple roots, if the user picked more than one thing
+ * to drop, and failed items to read */
+export type UploadDroppedJobs = {
+  roots: UploadJobRoot[];
+  failed?: UploadItem[];
+};
+
 export const getFilesTree = (
   event: Event & { dataTransfer: DataTransfer },
-  fcb?: (tree: any, documentsCount: number, totalSize: number) => void,
+  fcb?: (file: File) => void,
 ): Promise<FileTreeObject> => {
   return new Promise<FileTreeObject>(function (resolve) {
     function newDirectoryApi(input: DataTransfer, cb: (files?: File[], paths?: string[]) => void) {
@@ -82,6 +113,83 @@ export const getFilesTree = (
         files: any[] = [],
         rootPromises: any[] = [];
 
+      const mapFileEntryToUploadItem = (entry: FileSystemFileEntry): UploadItem => ({
+          isFile: true,
+          entry,
+          getFile: () => new Promise((resolve, reject) => entry.file(resolve, reject)),
+        });
+
+      const mapDirectoryEntryToUploadItem = (entry: FileSystemDirectoryEntry, error?: DOMException): UploadItem =>
+        ({ isFile: false, entry, error });
+
+      const mapEntryToUploadItem = (entry: FileSystemEntry): UploadItem => entry.isFile
+          ? mapFileEntryToUploadItem(entry as FileSystemFileEntry)
+          : mapDirectoryEntryToUploadItem(entry as FileSystemDirectoryEntry);
+
+      const mapDirectoryEntryToUploadItemsDeep = (entry: FileSystemDirectoryEntry): Promise<UploadItem[]> =>
+        new Promise((resolve, _reject_but_should_always_resolve_with_failed_instead) => {
+          const reader = entry.createReader();
+          reader.readEntries(
+            (entries) => resolve(entries.length
+              ? Promise.all(entries.flatMap(e => mapEntryToUploadItemsDeep(e)) as unknown as UploadItem[])
+              : [mapDirectoryEntryToUploadItem(entry)])
+            ,
+            (err) => resolve([mapDirectoryEntryToUploadItem(entry, err)])
+          )
+        });
+
+      const mapEntryToUploadItemsDeep = async (entry: FileSystemEntry): Promise<UploadItem[]> => (entry.isFile
+          ? [ await mapFileEntryToUploadItem(entry as FileSystemFileEntry) ]
+          : await mapDirectoryEntryToUploadItemsDeep(entry as FileSystemDirectoryEntry)).flat();
+
+      const mapRootEntryToUploadJob = async (dtitem: DataTransferItem): Promise<UploadJobRoot> => {
+        const entry = dtitem.webkitGetAsEntry()!;
+        return {
+          titleItem: mapEntryToUploadItem(entry),
+          items: await mapEntryToUploadItemsDeep(entry),
+        };
+      }
+
+      /**
+       * From a single drop event, descend items, then sum up in flat lists:
+       * - roots for ui display (corresponds to an item picked by the user when starting the drag)
+       *     - each has a title entry, and a list of items to upload and empty folders to create
+       *     - if a root is included it has at least one non failed operation
+       * - failures of descending those roots
+       *
+       * Based on:
+       *   - https://caniuse.com/?search=webkitGetAsEntry
+       *   - https://github.com/leonadler/drag-and-drop-across-browsers (2017)
+       *
+       * @param items the DataTransferItemList from the browser drop event
+       * @returns A list of roots in the format of {@link UploadDroppedJobs}
+       */
+      async function jobsFromDrop(items: DataTransferItemList): Promise<UploadDroppedJobs> {
+        const allRoots = (await Promise.all([...items].map(i => mapRootEntryToUploadJob(i))));
+        const failed = allRoots.flatMap(r => r.items.filter(e => !!e.error));
+        const roots = allRoots.filter(r => r.items.some(e => !e.error));
+        sortNaturally(roots, r => r.titleItem.entry.fullPath);
+        return { roots, failed: failed.length ? failed : undefined, };
+      }
+
+      const uploadItemToString = (item: UploadItem, prefix = "") => //TODO NONONO
+        `${prefix}${item.isFile ? "📄" : "📁"} ${item.entry.fullPath}${item.error ? " (Error: " + (item.error.stack || item.error) + ")" : ""}`;
+
+      jobsFromDrop(items).then(uploadJobs => {
+        if (uploadJobs.failed) {
+          console.warn(`Failed (${uploadJobs.failed.length}) :`);
+          uploadJobs.failed.forEach(i => console.log(uploadItemToString(i, "  ")));
+        }
+        for (const root of uploadJobs.roots) {
+          console.info("🎯", uploadItemToString(root.titleItem));
+          root.items.forEach(i => console.log(uploadItemToString(i, "    ")));
+        }
+        if (uploadJobs.roots.length && fcb) {
+          const first = uploadJobs.roots.map(r => r.titleItem).filter(i => i.isFile)[0] as UploadFileItemSpecific;
+          first.getFile().then(f => fcb(f));
+        }
+    }, e => console.error(e));
+
       function readEntries(entry: any, reader: any, oldEntries: any, cb: any) {
         const dirReader = reader || entry.createReader();
         dirReader.readEntries(function (entries: any) {
@@ -173,7 +281,6 @@ export const getFilesTree = (
         });
       });
 
-      fcb && fcb(tree, documents_number, total_size);
       resolve({ tree, documentsCount: documents_number, totalSize: total_size });
     };
 
@@ -196,7 +303,7 @@ export const getFilesTree = (
         cb(event, [], []);
       }
     } else {
-      fcb && fcb([(event.target as any).files[0]], 1, (event.target as any).files[0].size);
+      fcb && fcb((event.target as any).files[0]);
       resolve({
         tree: (event.target as any).files[0],
         documentsCount: 1,

tdrive/frontend/src/app/views/client/body/drive/browser.tsx

@@ -306,8 +306,6 @@ export default memo(
                 parentId,
               });
             }}
-            onDragOver={handleDragOver}
-            onDrop={handleDrop}
             disabled={inTrash || access === 'read'}
           >
             {role == 'admin' && <UsersModal />}

tdrive/frontend/src/app/views/client/popup/UserParameter/UserParameter.jsx

@@ -2,7 +2,6 @@
 /* eslint-disable react/jsx-key */
 import { Component } from 'react';
 
-import { getFilesTree } from '@components/uploads/file-tree-utils';
 import Collections from '@deprecated/CollectionsV1/Collections/Collections.js';
 import popupManager from '@deprecated/popupManager/popupManager.js';
 import currentUserService from '@deprecated/user/CurrentUser';

tdrive/frontend/src/app/views/client/popup/UserParameter/UserParameter_old.jsx

@@ -60,16 +60,15 @@ export default class UserParameter extends Component {
     // eslint-disable-next-line @typescript-eslint/no-this-alias
     var that = this;
     event.preventDefault();
-    getFilesTree(event, function (tree) {
-      var first = tree[Object.keys(tree)[0]];
-      if (first.constructor.name !== 'Object') {
-        //A file
+    getFilesTree(event, function (jobs) {
+      const first = jobs.roots.map(r => r.titleItem).filter(i => i.isFile)[0];
+      if (first) {
         var reader = new FileReader();
         reader.onload = function (e) {
           that.thumbnail.style.backgroundImage = "url('" + e.target.result + "')";
         };
         that.setState({ thumbnail: first });
-        reader.readAsDataURL(first);
+        first.getFile().then(file => reader.readAsDataURL(first));
       }
     });
   }

tdrive/frontend/src/utils/array.ts

@@ -0,0 +1,15 @@
+/**
+ * Sort the provided `array` by using the `getter` on each item to return a value, which
+ * is compared in a way that humans find natural (case and diacritic insensitive, ordinal
+ * for numeric segments)
+ */
+export function sortNaturally(array: string[]): string[];
+export function sortNaturally<TElement>(array: TElement[], getter: (x: TElement) => string): TElement[];
+export function sortNaturally<TElement>(array: TElement[], getter?: (x: TElement) => string): TElement[] {
+  const get = getter ?? ((x) => x as string);
+  array.sort((a, b) => get(a).localeCompare(get(b), undefined, {
+      numeric: true,
+      sensitivity: 'base'
+    }));
+  return array;
+}