SPB Git

Repository Public License

05-csv-cleanup-and-charts.yaml 6.6 KB 153 lines yaml

1# =============================================================================2# SP BOUCHER | SOURCE FILE3# =============================================================================4# Project      : ORIX OS Automation5# File         : Workflows/Examples/05-csv-cleanup-and-charts.yaml6# Author       : Simon-Pierre Boucher | https://spboucher.ai7# Contact      : support@spboucher.ai8# Copyright    : (c) 2026 Simon-Pierre Boucher9# License      : Proprietary, all rights reserved (see license file)10# =============================================================================11# END OF SP BOUCHER SOURCE HEADER12# =============================================================================1314# ORIX OS Automation15# File: Workflows/Examples/05-csv-cleanup-and-charts.yaml16# Description: Built-in automation: cleans a CSV file with a small Python script and draws an SVG bar chart of its first numeric column.17#18# Author: Simon-Pierre Boucher19# Contact: contact@spboucher.ai20#21# Limitations22# - The Python script uses the standard library only (no pandas, no matplotlib): the chart is a simple SVG bar chart23#   of the first column whose values are all numeric, limited to the first 40 rows.24# - Cleaning means: trim cells, drop empty rows, pad/cut rows to the header width, drop exact duplicate rows. Nothing smarter.25# - The script runs in a sandbox: it can read the input file and write the output folder, and has no network.26# - python.run uses ORIX's own virtual environment, created on first use from a python3 found on this Mac (Xcode27#   command-line tools or Homebrew). Without any python3 the step fails with that reason; system Python is never modified.28#   The first run takes a few seconds longer while the environment is created.29# - The input must be UTF-8 (a byte-order mark is accepted); the cleaned file is always comma-separated UTF-8.30apiVersion: orix.ai/v131kind: Automation32metadata:33  id: csv-cleanup-and-charts34  name: CSV cleanup and chart generation35  description: >-36    Cleans a CSV file (trimmed cells, no empty or duplicate rows, consistent width) with a small Python37    script and draws an SVG bar chart of its first numeric column, then checks both files.38  tags: [csv, python, chart, data]39spec:40  triggers:41    - type: manual42  parameters:43    - name: inputFile44      type: path45      default: ~/ORIX Samples/csv-cleanup-and-charts/input.csv46      description: CSV file to clean (first row is the header).47    - name: outputFolder48      type: path49      default: ~/ORIX Samples/csv-cleanup-and-charts/Output50      description: Folder that receives cleaned.csv and chart.svg.51  permissions:52    filesystem:53      read: ["${{ params.inputFile }}", "${{ params.outputFolder }}"]54      write: ["${{ params.outputFolder }}"]55    scripts: [python]56  budget:57    maxDurationSeconds: 60058    maxToolCalls: 2059  steps:60    - id: inspect61      name: Read the original file62      tool: documents.csvRead63      with:64        path: ${{ params.inputFile }}65        header: "yes"66        limit: 567    - id: prepare68      tool: filesystem.mkdir69      with:70        path: ${{ params.outputFolder }}71    - id: clean72      name: Clean and chart73      needs: [inspect, prepare]74      tool: python.run75      with:76        args: ["${{ params.inputFile }}", "${{ params.outputFolder }}"]77        cwd: ${{ params.outputFolder }}78        timeoutSeconds: 30079        code: |80          import csv, os, sys8182          src = os.path.expanduser(sys.argv[1])83          out = os.path.expanduser(sys.argv[2])8485          with open(src, newline="", encoding="utf-8-sig") as f:86              rows = [[cell.strip() for cell in row] for row in csv.reader(f)]87          rows = [row for row in rows if any(row)]88          if not rows:89              sys.exit("the input file has no rows")90          header, body = rows[0], rows[1:]9192          seen, clean = set(), []93          for row in body:94              row = (row + [""] * len(header))[:len(header)]95              if tuple(row) not in seen:96                  seen.add(tuple(row))97                  clean.append(row)9899          with open(os.path.join(out, "cleaned.csv"), "w", newline="", encoding="utf-8") as f:100              csv.writer(f).writerows([header] + clean)101102          def number(text):103              try:104                  return float(text.replace(",", ""))105              except ValueError:106                  return None107108          column = next((i for i in range(len(header)) if clean and all(number(r[i]) is not None for r in clean)), None)109          points = [] if column is None else [(r[0], number(r[column])) for r in clean[:40]]110          top = max([abs(v) for _, v in points] + [1.0])111          bar, gap, height = 18, 6, 220112          width = max(320, 40 + len(points) * (bar + gap))113          title = "No numeric column found" if column is None else header[column]114          parts = ['<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d">' % (width, height + 60),115                   '<text x="20" y="24" font-family="Helvetica" font-size="14">%s</text>' % title.replace("&", "&amp;").replace("<", "&lt;")]116          for i, (_, value) in enumerate(points):117              h = abs(value) / top * height118              parts.append('<rect x="%d" y="%.1f" width="%d" height="%.1f" fill="#6c5ce7"/>' % (20 + i * (bar + gap), 40 + height - h, bar, h))119          parts.append("</svg>")120          with open(os.path.join(out, "chart.svg"), "w", encoding="utf-8") as f:121              f.write("\n".join(parts))122123          print("rows_in=%d rows_out=%d charted_column=%s bars=%d" % (len(body), len(clean), title, len(points)))124    - id: cleaned125      name: Read the cleaned file back126      needs: [clean]127      tool: documents.csvRead128      with:129        path: ${{ params.outputFolder }}/cleaned.csv130        header: "yes"131        limit: 5132    - id: check133      needs: [cleaned]134      verify:135        - equals: ["${{ steps.clean.output.exitCode }}", 0]136        - fileNonEmpty: ${{ params.outputFolder }}/cleaned.csv137        - fileContains: ${{ params.outputFolder }}/chart.svg138          text: <svg139        - truthy: ${{ steps.cleaned.output.totalRows > 0 }}140        - truthy: ${{ steps.cleaned.output.totalRows <= steps.inspect.output.totalRows }}141    - id: result142      needs: [check]143      output:144        cleaned: ${{ params.outputFolder }}/cleaned.csv145        chart: ${{ params.outputFolder }}/chart.svg146        rowsBefore: ${{ steps.inspect.output.totalRows }}147        rows: ${{ steps.cleaned.output.totalRows }}148        columns: ${{ steps.cleaned.output.columns }}149        log: ${{ trim(default(steps.clean.output.stdout, "")) }}150      artifacts:151        - ${{ params.outputFolder }}/cleaned.csv152        - ${{ params.outputFolder }}/chart.svg153
Files main
Menu

Theme