SPB Git

Repository Public License

12-document-metadata-catalog.yaml 3.9 KB 106 lines yaml

1# ORIX OS Automation2# File: Workflows/Examples/12-document-metadata-catalog.yaml3# Description: Built-in automation: walks a folder and writes size, modification date and SHA-256 of every file to a CSV catalogue.4#5# Author: Simon-Pierre Boucher6# Contact: contact@spboucher.ai7#8# Limitations9# - Filesystem metadata only (size, modification date, content type) plus a SHA-256 per file; document-internal metadata10#   (PDF author, EXIF, …) is not read here — see the "PDF to structured data and report" example for PDFs.11# - The folder is walked 16 levels deep; hidden files (dot files) are skipped and symbolic links are not followed12#   or catalogued.13# - Hashing large folders takes time; four files are processed at once. One listing returns at most 2 000 files and14#   the tool-call budget (two calls per file) is sized for that: a larger folder makes the verification fail rather15#   than produce a silently incomplete catalogue.16# - The catalogue is rewritten in full at every run (the previous file is first copied to the run's artifacts); no history is kept.17apiVersion: orix.ai/v118kind: Automation19metadata:20  id: document-metadata-catalog21  name: Document metadata extraction and cataloging22  description: >-23    Walks a folder, reads size, modification date and SHA-256 of every file and writes one catalogue row24    per file to a CSV, then checks that the catalogue has as many rows as files found.25  tags: [documents, metadata, catalog, csv]26spec:27  triggers:28    - type: manual29  parameters:30    - name: documentsFolder31      type: path32      default: ~/ORIX Samples/document-metadata-catalog/Documents33      description: Folder to catalogue (recursively).34    - name: catalogFile35      type: path36      default: ~/ORIX Samples/document-metadata-catalog/catalog.csv37      description: CSV file to write.38  permissions:39    filesystem:40      read: ["${{ params.documentsFolder }}", "${{ params.catalogFile }}"]41      write: ["${{ params.catalogFile }}"]42  budget:43    maxDurationSeconds: 180044    maxToolCalls: 500045  steps:46    - id: list47      tool: filesystem.list48      with:49        path: ${{ params.documentsFolder }}50        recursive: true51        maxDepth: 1652        limit: 200053    - id: files54      needs: [list]55      transform: ${{ filter(steps.list.output.entries, e => e.type == "file") }}56    - id: describe57      name: Describe each file58      needs: [files]59      foreach: ${{ steps.files.output }}60      as: doc61      concurrency: 462      collect: row63      steps:64        - id: meta65          tool: filesystem.metadata66          with:67            path: ${{ doc.path }}68        - id: hash69          tool: filesystem.checksum70          with:71            path: ${{ doc.path }}72        - id: row73          needs: [meta, hash]74          transform: ${{ [doc.name, lower(extname(doc.name)), steps.meta.output.size, steps.meta.output.modified, default(steps.meta.output.contentType, ""), steps.hash.output.digest] }}75    - id: catalog76      name: Write the catalogue77      needs: [describe]78      tool: documents.csvWrite79      with:80        path: ${{ params.catalogFile }}81        columns: [path, extension, bytes, modified, contentType, sha256]82        rows: ${{ steps.describe.output.results }}83        overwrite: true84    - id: readBack85      needs: [catalog]86      tool: documents.csvRead87      with:88        path: ${{ params.catalogFile }}89        header: "yes"90        limit: 191    - id: check92      needs: [readBack]93      verify:94        - fileNonEmpty: ${{ params.catalogFile }}95        - truthy: ${{ !steps.list.output.truncated }}96        - equals: ["${{ steps.describe.output.count }}", "${{ len(steps.files.output) }}"]97        - equals: ["${{ steps.readBack.output.totalRows }}", "${{ len(steps.files.output) }}"]98    - id: result99      needs: [check]100      output:101        catalog: ${{ params.catalogFile }}102        files: ${{ len(steps.files.output) }}103        totalBytes: ${{ sum(steps.describe.output.results, r => r[2]) }}104      artifacts:105        - ${{ params.catalogFile }}106
Files main
Menu

Theme