> ## Documentation Index
> Fetch the complete documentation index at: https://developer.upsun.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Build a log ingestion project on Upsun Cloud with Vector and OpenSearch

> Collect Fastly and Upsun logs with Vector, search them in OpenSearch Dashboards, and configure authentication, retention, and optional object storage archives.


export const PostMeta = ({data = {}}) => {
  const {author, date} = data;
  const authors = Array.isArray(author) ? author : author ? [author] : [];
  const toSlug = value => String(value).toLowerCase().trim().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, '');
  const resolveAuthor = slug => {
    const entry = AUTHOR_MAP[slug] || ({});
    const name = entry.name || slug;
    const github = entry.github || null;
    const url = `/posts/authors/${toSlug(slug)}`;
    const avatarUrl = github ? `https://github.com/${github}.png?size=64` : null;
    return {
      name,
      url,
      avatarUrl
    };
  };
  const formattedDate = date ? new Date(date).toLocaleDateString('en-US', {
    year: 'numeric',
    month: 'long',
    day: 'numeric'
  }) : null;
  if (authors.length === 0 && !formattedDate) return null;
  const AUTHOR_MAP = {
    "aaron-collier": {
      "name": "Aaron Collier"
    },
    "aaron-dudenhofer": {
      "name": "Aaron Dudenhofer"
    },
    "aaron-porter": {
      "name": "Aaron Porter"
    },
    "adriaan-odendaal": {
      "name": "Adriaan Odendaal"
    },
    "ajmal": {
      "name": "Ajmal Siddiqui"
    },
    "akalipetis": {
      "name": "Antonis Kalipetis"
    },
    "alexander-varwijk": {
      "name": "Alexander Varwijk"
    },
    "alicia-bevilacqua": {
      "name": "Alicia Bevilacqua"
    },
    "amelie-deguerry": {
      "name": "Amelie Deguerry"
    },
    "anacidre": {
      "name": "Ana Cidre",
      "linkedin": "https://www.linkedin.com/in/ana-cidre"
    },
    "andoni": {
      "name": "Andoni Auzmendi"
    },
    "andrei-taranu": {
      "name": "Andrei (Alex) Taranu",
      "linkedin": "https://www.linkedin.com/in/andrei-alex-taranu/"
    },
    "andrew-baxter": {
      "name": "Andrew Baxter"
    },
    "andrew-melck": {
      "name": "Andrew Melck"
    },
    "antoine-crochet-damais": {
      "name": "Antoine Crochet Damais"
    },
    "augustin-delaporte": {
      "name": "Augustin Delaporte",
      "linkedin": "https://www.linkedin.com/in/augustindelaporte/"
    },
    "branislav-bujisic": {
      "name": "Branislav Bujisic"
    },
    "carl-smith": {
      "name": "Carl Smith"
    },
    "caroline-leroy": {
      "name": "Caroline Leroy"
    },
    "cati-mayer": {
      "name": "Cati Mayer"
    },
    "catplat": {
      "name": "C Trinkwon"
    },
    "ceelolulu": {
      "name": "Celeste van der Watt"
    },
    "chadwcarlson": {
      "name": "Chad Carlson",
      "github": "chadwcarlson",
      "linkedin": "https://www.linkedin.com/in/chadwcarlson"
    },
    "charles-moore": {
      "name": "Charles Moore",
      "linkedin": "https://www.linkedin.com/in/charles-g-moore/"
    },
    "chris-ward": {
      "name": "Chris Ward"
    },
    "chris-yates": {
      "name": "Chris Yates"
    },
    "christian-sieber": {
      "name": "Christian Sieber"
    },
    "christopher-lockheardt": {
      "name": "Christopher Lockheardt"
    },
    "christopher-skene": {
      "name": "Christopher Skene"
    },
    "chuck-morgan": {
      "name": "Chuck Morgan"
    },
    "corey-dockendorf": {
      "name": "Corey Dockendorf"
    },
    "crell": {
      "name": "Crell"
    },
    "damz": {
      "name": "Damz"
    },
    "dan-morrison": {
      "name": "Dan Morrison"
    },
    "davidbonachera": {
      "name": "David Bonachera",
      "github": "davidbonachera",
      "linkedin": "https://www.linkedin.com/in/davidbonachera"
    },
    "dereliahmet1": {
      "name": "Ahmet Faruk Dereli"
    },
    "devicezero": {
      "name": "Jonas Kröger",
      "github": "devicezero",
      "linkedin": "https://www.linkedin.com/in/jonaskroeger/"
    },
    "doug-goldberg": {
      "name": "Doug Goldberg"
    },
    "duncan-naves": {
      "name": "Duncan Naves",
      "github": "duncannaves",
      "linkedin": "https://www.linkedin.com/in/duncan-naves-a94423aa"
    },
    "erika-bustamante": {
      "name": "Erika Bustamante"
    },
    "fabpot": {
      "name": "Fabien Potencier"
    },
    "flovntp": {
      "name": "Florent Huck",
      "github": "flovntp",
      "linkedin": "https://www.linkedin.com/in/florenthuck"
    },
    "fred-plais": {
      "name": "Fred Plais"
    },
    "gauthier-garnier": {
      "name": "Gauthier Garnier"
    },
    "gilzow": {
      "name": "Paul Gilzow"
    },
    "gmoigneu": {
      "name": "Guillaume Moigneu",
      "github": "gmoigneu",
      "linkedin": "https://www.linkedin.com/in/guillaumemoigneu/"
    },
    "gregqualls": {
      "name": "Greg Qualls"
    },
    "guguss": {
      "name": "Augustin Delaporte"
    },
    "haylee-millar": {
      "name": "Haylee Millar"
    },
    "ivana-kotur": {
      "name": "Ivana Kotur"
    },
    "jackrabbithanna": {
      "name": "Mark Hanna",
      "github": "jackrabbithanna"
    },
    "jared-wright": {
      "name": "Jared Wright",
      "github": "jww-sh",
      "linkedin": "https://www.linkedin.com/in/jaredwaynewright"
    },
    "jessica-orozco": {
      "name": "Jessica Orozco"
    },
    "joey-stanford": {
      "name": "Joey Stanford"
    },
    "john-grubb": {
      "name": "John Grubb"
    },
    "jonas-kruger": {
      "name": "Jonas Kruger"
    },
    "kathryn-frazer": {
      "name": "Kathryn Frazer"
    },
    "kemiojo": {
      "name": "Kemi Elizabeth Ojogbede"
    },
    "kieronsambrook-smith": {
      "name": "Kieronsambrook Smith"
    },
    "laurent-arnoud": {
      "name": "Laurent Arnoud",
      "linkedin": "https://www.linkedin.com/in/laurent-arnoud-861b44121/"
    },
    "letoya-boyne": {
      "name": "Letoya Boyne"
    },
    "lolautruche": {
      "name": "Jérôme Vieilledent"
    },
    "lyly-lepinay": {
      "name": "Lyly Lepinay"
    },
    "manauwar-alam": {
      "name": "Manauwar Alam"
    },
    "marc-antoine-porri": {
      "name": "Marc Antoine Porri"
    },
    "maria-antinkaapo": {
      "name": "Maria Antinkaapo"
    },
    "maria-de-anton": {
      "name": "Maria De Anton"
    },
    "mark-dorison": {
      "name": "Mark Dorison"
    },
    "markus-hausammann": {
      "name": "Markus Hausammann"
    },
    "mary-thomas": {
      "name": "Mary Thomas"
    },
    "mathias-bolt-lesniak": {
      "name": "Mathias Bolt Lesniak"
    },
    "mathieu-strauch": {
      "name": "Mathieu Strauch"
    },
    "matthias-van-woensel": {
      "name": "Matthias Van Woensel",
      "linkedin": "https://www.linkedin.com/in/matthias-van-woensel-267a069"
    },
    "maz-mohammadi": {
      "name": "Maz Mohammadi"
    },
    "michael-sharp": {
      "name": "Michael Sharp"
    },
    "mupsi": {
      "name": "Marine Gandy"
    },
    "natalie-harper": {
      "name": "Natalie Harper"
    },
    "ngommenginger": {
      "name": "Nicolas Gommenginger",
      "linkedin": "https://www.linkedin.com/in/nicolas-gommenginger"
    },
    "nicholas-bennison": {
      "name": "Nicholas Bennison"
    },
    "nicholas-vahalik": {
      "name": "Nicholas Vahalik"
    },
    "nick-hardiman": {
      "name": "Nick Hardiman"
    },
    "nickanderegg": {
      "name": "Nickanderegg"
    },
    "nicolas-grekas": {
      "name": "Nicolas Grekas",
      "github": "nicolas-grekas",
      "linkedin": "https://www.linkedin.com/in/nicolasgrekas/"
    },
    "niti-malwade": {
      "name": "Niti Malwade"
    },
    "opensocialteam": {
      "name": "Opensocialteam"
    },
    "ori-pekelman": {
      "name": "Ori Pekelman"
    },
    "otavio-santana": {
      "name": "Otavio Santana"
    },
    "palwandi": {
      "name": "Pawan Alwandi",
      "github": "pawpy",
      "linkedin": "https://www.linkedin.com/in/pawanalwandi"
    },
    "patrick-boest": {
      "name": "Patrick Boest"
    },
    "patrick-dawkins": {
      "name": "Patrick Dawkins",
      "github": "pjcdawkins",
      "linkedin": "https://www.linkedin.com/in/patrickdawkins"
    },
    "patrick-klima": {
      "name": "Patrick Klima"
    },
    "pjcdawkins": {
      "name": "Pjcdawkins"
    },
    "prineet-kaurbhurji": {
      "name": "Prineet Kaurbhurji"
    },
    "quentin-sinig": {
      "name": "Quentin Sinig"
    },
    "ralt": {
      "name": "Florian Margaine",
      "github": "ralt",
      "linkedin": "https://www.linkedin.com/in/florian-margaine-43971136"
    },
    "ramanathanramakrishnamurthy": {
      "name": "Ramanathanramakrishnamurthy"
    },
    "remi-lejeune": {
      "name": "Rémi Lejeune"
    },
    "ribel": {
      "name": "Taras Kruts"
    },
    "robert-douglass": {
      "name": "Robert Douglass"
    },
    "rudy-weber": {
      "name": "Rudy Weber"
    },
    "ryan-hicks": {
      "name": "Ryan Hicks"
    },
    "sabri-helal": {
      "name": "Sabri Helal"
    },
    "savannah-bergeron": {
      "name": "Savannah Bergeron"
    },
    "shannon-vettes": {
      "name": "Shannon Vettes"
    },
    "shawn-ogasawara": {
      "name": "Shawn Ogasawara",
      "linkedin": "https://www.linkedin.com/in/shawn-ogasawara-83a9a0/"
    },
    "shawna-spoor": {
      "name": "Shawna Spoor"
    },
    "shedrack-akintayo": {
      "name": "Shedrack Akintayo"
    },
    "simon-ruggier": {
      "name": "Simon Ruggier"
    },
    "sophie-van-der-kindere": {
      "name": "Sophie Van Der Kindere"
    },
    "stefanos-thampis": {
      "name": "Stefanos Thampis"
    },
    "stephen-weinberg": {
      "name": "Stephen Weinberg"
    },
    "sukhman-virk": {
      "name": "Sukhman Virk"
    },
    "sumaira-nazir": {
      "name": "Sumaira Nazir"
    },
    "sumer": {
      "name": "Sümer Cip",
      "noindex": true,
      "github": "sumerc",
      "linkedin": "https://www.linkedin.com/in/sumer-cip/"
    },
    "syed-raza": {
      "name": "Syed Raza"
    },
    "tamara-bacchia": {
      "name": "Tamara Bacchia"
    },
    "tara-arnold": {
      "name": "Tara Arnold"
    },
    "theosakamg": {
      "name": "Mickael Gaillard",
      "github": "theosakamg"
    },
    "thomasdiluccio": {
      "name": "Thomas di Luccio"
    },
    "tim-anderson": {
      "name": "Tim Anderson"
    },
    "tom-helmer-hansen": {
      "name": "Tom Helmer Hansen"
    },
    "tylermills": {
      "name": "Tyler Mills"
    },
    "upsun": {
      "name": "Upsun"
    },
    "veronika-tolkachova": {
      "name": "Veronika Tolkachova",
      "linkedin": "https://www.linkedin.com/in/veronika-tolkachova-169167a2"
    },
    "vince-parker": {
      "name": "Vince Parker"
    },
    "vinnie-russo": {
      "name": "Vincenzo Russo"
    },
    "vrobert78": {
      "name": "Vincent Robert",
      "github": "vrobert78",
      "linkedin": "https://www.linkedin.com/in/vincent-robert-498a883"
    },
    "yuriy-babenko": {
      "name": "Yuriy Babenko"
    },
    "yuriy-gerasimov": {
      "name": "Yuriy Gerasimov"
    }
  };
  return <div className="post-meta">
      {(authors.length > 0 || formattedDate) && <div className="post-meta-info">
          {authors.length > 0 && <div className="post-meta-authors">
              {authors.map(slug => {
    const {name, url, avatarUrl} = resolveAuthor(slug);
    const inner = <>
                    {avatarUrl && <img src={avatarUrl} alt={name} className="post-meta-avatar" />}
                    <span className="post-meta-author-name">{name}</span>
                  </>;
    return url ? <a key={slug} href={url} className="post-meta-author">
                    {inner}
                  </a> : <span key={slug} className="post-meta-author">{inner}</span>;
  })}
            </div>}
          {authors.length > 0 && formattedDate && <span className="post-meta-separator" aria-hidden="true">·</span>}
          {formattedDate && <span className="post-meta-date">{formattedDate}</span>}
        </div>}
    </div>;
};

<PostMeta data={{ author: ["charles-moore"], date: "2026-10-13T09:00:00Z" }} />

## Overview

This guide builds a dedicated Upsun Cloud project that receives logs over HTTPS, parses them with [Vector](https://vector.dev), and stores them in OpenSearch for search and analysis. Optionally, Vector also writes a compressed copy to Upsun object storage so you can keep OpenSearch retention short and archive cheaply.

The project accepts two kinds of logs:

* **Fastly CDN logs**, streamed from one or more Fastly services through an HTTPS logging endpoint, in a structured JSON format.
* **Upsun application and service logs**, forwarded from one or many Upsun projects through the `httplog` log forwarding integration.

```text theme={null}
  Fastly services ──── HTTPS JSON ───┐
  Upsun project A ──── httplog ──────┤
  Upsun project B ──── httplog ──────┤
                                     ▼
                              Upsun router
                    ┌────────────────┴────────────────┐
           logs.example.com              dashboards.logs.example.com
                    ▼                                  ▼
               Vector app                Caddy + OpenSearch Dashboards app
                    │                                  │
                    ├──────────► OpenSearch service ◄──┘
                    │
                    └ ─ ─ optional ─ ─► Upsun object storage
```

Everything runs inside one Upsun project with two apps: Vector receives logs, and OpenSearch Dashboards gives you a browser UI to search them. The router terminates TLS for both. OpenSearch and object storage are reachable only on the project's internal network.

**Single project or many?** The setup is the same either way. Each source project or Fastly service points at the same ingestion URL, and Vector tags every event with its source so you can filter by project, environment, or Fastly service in OpenSearch.

## Prerequisites

You need the following before you start:

* An Upsun organization where you can create projects, and the [Upsun CLI](/cli/install) installed and logged in (`upsun login`).
* Project admin access on every Upsun project whose logs you want to forward. Real-time log forwarding is a paid feature on Upsun, so confirm it's enabled for those projects.
* An Upsun project served through Fastly. Upsun Support adds the Fastly logging endpoint for you through a ticket (Step 6).
* A domain or subdomain for the ingestion endpoint, such as `logs.example.com`, that you can point at Upsun.
* `git` and `curl` on your workstation.

Plan for volume. OpenSearch is the largest cost driver. Estimate daily log volume first: a busy Fastly service can produce several GB of JSON per day, and OpenSearch disk usage depends on your mappings, replicas, and indexed fields. The optional object storage archive, near the end of this guide, exists to keep this number small.

## Step 1: Create the Flex project

Create an empty project in the same region as the projects and Fastly POPs you'll send the most logs from. That keeps latency and egress down.

```sh theme={null}
  upsun project:create
```

The CLI prompts for the organization, title, region, and default branch. It looks like this:

```text theme={null}
  Enter a number to choose an organization (-o):
    [0] My Projects
    [1] Example Flex Org
    [2] Another Org
   > 1
  Creating a project under the organization Example Flex Org

  * Project title (--title)
  > Log ingestion

  * Region (--region)
    [ca-1.platform.sh] Montreal, Canada (AWS)
    [eu-3.platform.sh] Dublin, Ireland (AWS)
    [us-3.platform.sh] Moses Lake, United States (Azure)
    ...
  > eu-3.platform.sh

  Default branch (--default-branch)
  Default: main
  >

  Are you sure you want to continue? [Y/n] y

  The project is now ready!
    Region: eu-3.platform.sh
    Project ID: abcdefgh1234567
    Project title: Log ingestion
    Git URL: abcdefgh1234567@git.eu-3.platform.sh:abcdefgh1234567.git
```

Clone the new project using the project ID from the output:

```sh theme={null}
  upsun get <project-id>
```

The output looks like this:

```text theme={null}
  Selected default environment: main (type: production)
  Directory (default: log-ingestion):
  Creating project directory: log-ingestion
    Initialized empty Git repository in /Users/you/projects/log-ingestion/.git/
  Your project has been initialized and connected to Upsun!
  Commit and push to the main branch of the upsun Git remote, and Upsun will build your project automatically.
```

Then move into the new directory with `cd log-ingestion`.

The clone already contains a `.upsun/local/` folder. That's the CLI's local cache for this project, so keep it out of git before you add anything else:

```sh theme={null}
  echo ".upsun/local/" >> .gitignore
```

By the end of this guide the repository will look like this:

```text theme={null}
  log-ingestion/
  ├── .upsun/
  │   └── config.yaml                  # apps, services, routes
  ├── vector/
  │   ├── vector.yaml                  # sources, transforms, sinks
  │   └── public/
  │       └── .well-known/fastly/logging/challenge
  └── dashboards/
      ├── Caddyfile                    # HTTP Basic auth in front of Dashboards
      ├── opensearch_dashboards.yml    # Dashboards settings
      └── supervisor.conf              # Caddy and Dashboards processes
```

Create the folders and empty files now, from the project root:

```sh theme={null}
  mkdir -p .upsun vector/public/.well-known/fastly/logging dashboards
  touch .upsun/config.yaml vector/vector.yaml \
    dashboards/opensearch_dashboards.yml dashboards/Caddyfile dashboards/supervisor.conf
```

The `challenge` file answers Fastly's ownership check for HTTPS logging endpoints (see Step 6).

## Step 2: Define apps, services, and routes

The project has two apps and one service. Both apps use the [default OpenSearch relationship](/docs/configure-apps/image-properties/relationships), and [mounts](/docs/configure-apps/image-properties/mounts) keep their data writable at runtime. The [build hooks](/docs/configure-apps/hooks) download pinned releases of Vector, OpenSearch Dashboards, and Caddy. Their archives are kept in `$PLATFORM_CACHE_DIR` to reuse them across builds. Create `.upsun/config.yaml`:

```yaml title=".upsun/config.yaml" theme={null}
  applications:
    vector:
      type: "python:3.12"          # base image only; Vector is a static binary
      source:
        root: "vector"
      build:
        flavor: none
      hooks:
        build: |
          set -e
          VECTOR_VERSION="0.58.0"
          ARCHIVE="$PLATFORM_CACHE_DIR/vector-${VECTOR_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
          if [ ! -f "$ARCHIVE" ]; then
            curl -sSfL "https://packages.timber.io/vector/${VECTOR_VERSION}/vector-${VECTOR_VERSION}-x86_64-unknown-linux-gnu.tar.gz" \
              -o "$ARCHIVE.tmp"
            mv "$ARCHIVE.tmp" "$ARCHIVE"
          fi
          TMP="$(mktemp -d)"
          tar xzf "$ARCHIVE" -C "$TMP"
          mkdir -p bin
          mv "$TMP"/vector-*/bin/vector bin/vector
          rm -rf "$TMP"
          ./bin/vector --version
      relationships:
        opensearch:
      mounts:
        "data":                       # Vector disk buffers and state
          source: "storage"
          source_path: "vector-data"
      web:
        upstream:
          socket_family: tcp
        commands:
          start: "./bin/vector --config vector.yaml --dangerously-allow-env-var-interpolation"
        locations:
          "/":
            passthru: true
            allow: false
          "/.well-known/fastly/logging":
            root: "public/.well-known/fastly/logging"
            passthru: false
            allow: true
            scripts: false

    dashboards:
      type: "nodejs:22"            # base image only; Dashboards ships its own Node
      source:
        root: "dashboards"
      dependencies:
        python3:
          supervisor: "4.3.0"
      build:
        flavor: none
      hooks:
        build: |
          set -e
          OSD_VERSION="3.8.0"        # must match your OpenSearch version (see Step 4)
          CADDY_VERSION="2.11.4"
          OSD_ARCHIVE="$PLATFORM_CACHE_DIR/opensearch-dashboards-${OSD_VERSION}-linux-x64.tar.gz"
          if [ ! -f "$OSD_ARCHIVE" ]; then
            curl -sSfL "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/${OSD_VERSION}/opensearch-dashboards-${OSD_VERSION}-linux-x64.tar.gz" \
              -o "$OSD_ARCHIVE.tmp"
            mv "$OSD_ARCHIVE.tmp" "$OSD_ARCHIVE"
          fi
          tar xzf "$OSD_ARCHIVE"
          mv "opensearch-dashboards-${OSD_VERSION}" osd
          ./osd/bin/opensearch-dashboards-plugin remove securityDashboards
          mkdir -p bin
          CADDY_ARCHIVE="$PLATFORM_CACHE_DIR/caddy_${CADDY_VERSION}_linux_amd64.tar.gz"
          if [ ! -f "$CADDY_ARCHIVE" ]; then
            curl -sSfL "https://github.com/caddyserver/caddy/releases/download/v${CADDY_VERSION}/caddy_${CADDY_VERSION}_linux_amd64.tar.gz" \
              -o "$CADDY_ARCHIVE.tmp"
            mv "$CADDY_ARCHIVE.tmp" "$CADDY_ARCHIVE"
          fi
          tar xzf "$CADDY_ARCHIVE" -C bin caddy
      relationships:
        opensearch:
      mounts:
        "osd/data":
          source: "storage"
          source_path: "osd-data"
      web:
        upstream:
          socket_family: tcp
        commands:
          start: |
            set -e
            export XDG_CONFIG_HOME=/tmp XDG_DATA_HOME=/tmp
            export DASHBOARDS_PASSWORD_HASH="$(./bin/caddy hash-password --plaintext "$DASHBOARDS_PASSWORD")"
            export OPENSEARCH_BASIC="$(printf '%s:%s' "$OPENSEARCH_USERNAME" "$OPENSEARCH_PASSWORD" | base64 | tr -d '\n')"
            exec supervisord -n -c supervisor.conf

  services:
    opensearch:
      type: "opensearch:3"
      configuration:
        authentication:
          enabled: true

  routes:
    "https://{default}/":
      type: upstream
      upstream: "vector:http"
    "https://dashboards.{default}/":
      type: upstream
      upstream: "dashboards:http"
```

Key points:

* [`socket_family: tcp`](/docs/configure-apps/image-properties/web#upstream) makes Upsun hand each app a `$PORT` to listen on. Both configs read it from the environment.
* [`locations`](/docs/configure-apps/image-properties/web#locations) on the Vector app serve the static Fastly challenge file directly and pass every other request to Vector.
* **`authentication.enabled: true`** turns on OpenSearch credentials. Upsun exposes them to both apps as `OPENSEARCH_USERNAME` and `OPENSEARCH_PASSWORD`, alongside `OPENSEARCH_HOST` and `OPENSEARCH_PORT`.
* **Object storage** isn't used by default. If you add the optional archive, you enable it with a resources command, and Upsun adds the `OBJECT_STORAGE_*` variables to the Vector app automatically.

Create `dashboards/supervisor.conf` to manage both processes:

```ini title="dashboards/supervisor.conf" theme={null}
  [supervisord]
  nodaemon=true
  logfile=/dev/stdout
  logfile_maxbytes=0
  pidfile=/tmp/supervisord.pid
  childlogdir=/tmp

  [program:dashboards]
  command=/app/osd/bin/opensearch-dashboards --config /app/opensearch_dashboards.yml
  directory=/app
  autostart=true
  autorestart=true
  startsecs=0
  stopasgroup=true
  killasgroup=true
  redirect_stderr=true
  stdout_logfile=/dev/stdout
  stdout_logfile_maxbytes=0

  [program:caddy]
  command=/app/bin/caddy run --config /app/Caddyfile --adapter caddyfile
  directory=/app
  autostart=true
  autorestart=true
  startsecs=0
  stopasgroup=true
  killasgroup=true
  redirect_stderr=true
  stdout_logfile=/dev/stdout
  stdout_logfile_maxbytes=0
```

Upsun's [web start command](/docs/configure-apps/image-properties/web#web-commands) runs Supervisor in the foreground. Upsun restarts Supervisor if it exits; Supervisor starts and restarts both Caddy and Dashboards. Both children inherit the environment prepared by the start command, including Upsun's `NODE_OPTIONS` heap setting. Their output goes to the app log.

Create `dashboards/opensearch_dashboards.yml`:

```yaml title="dashboards/opensearch_dashboards.yml" theme={null}
  server.host: "127.0.0.1"                  # only Caddy can reach Dashboards
  server.port: 5601
  opensearch.hosts: ["http://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}"]
  opensearch.username: "${OPENSEARCH_USERNAME}"
  opensearch.password: "${OPENSEARCH_PASSWORD}"
  opensearch.requestHeadersAllowlist: ["authorization"]   # Caddy fills this with the OpenSearch credentials
  path.data: "osd/data"
```

Then create `dashboards/Caddyfile`. Caddy listens on the Upsun port, asks for a username and password, and passes authenticated requests to Dashboards:

```text title="dashboards/Caddyfile" theme={null}
  {
    admin off
    auto_https off
  }

  :{$PORT} {
    basic_auth {
      {$DASHBOARDS_USER} {$DASHBOARDS_PASSWORD_HASH}
    }
    reverse_proxy 127.0.0.1:5601 {
      header_up Authorization "Basic {$OPENSEARCH_BASIC}"
    }
  }
```

Why Caddy rewrites the `Authorization` header. Upsun protects OpenSearch with plain HTTP basic auth, not the OpenSearch Security plugin. Dashboards uses `opensearch.username` only for its own startup tasks. For requests made on behalf of a person, it forwards that request's `Authorization` header. Caddy checks the person's Dashboards password, then swaps in the OpenSearch credentials, so every OpenSearch call is authenticated. Without the swap, Dashboards loads but every page returns `401` from OpenSearch.

<Warning>
  How access works. Opening `https://dashboards.logs.example.com/` makes your browser ask for the `DASHBOARDS_USER` username and password (you set them in Step 4). The ingestion endpoint isn't affected. Dashboards itself has no second login: it talks to OpenSearch with the platform-generated `OPENSEARCH_USERNAME` and `OPENSEARCH_PASSWORD`, so anyone with the HTTP auth password can see every log. Share it accordingly.
</Warning>

## Step 3: Configure Vector

Vector listens on one HTTPS route and sorts traffic by URL path. Fastly posts to `/fastly`, and each Upsun project posts to `/upsun/<project-id>`. Vector checks HTTP basic auth on every request.

First, store the ingestion credentials as sensitive project variables so they never live in git:

```sh theme={null}
  upsun variable:create --level project --name env:INGEST_USERNAME \
    --value "logshipper" --visible-build false --visible-runtime true
  upsun variable:create --level project --name env:INGEST_PASSWORD \
    --value "$(openssl rand -hex 24)" --sensitive true --visible-build false --visible-runtime true
```

These are separate from the Dashboards credentials (Step 4) and from the `OPENSEARCH_USERNAME` and `OPENSEARCH_PASSWORD` that Vector sends on every OpenSearch call.

Then create `vector/vector.yaml`:

```yaml title="vector/vector.yaml" theme={null}
  data_dir: "data"

  sources:
    ingest:
      type: http_server
      address: "0.0.0.0:${PORT}"
      path: "/"
      strict_path: false             # accept /fastly, /upsun/<id>, ...
      path_key: "_ingest_path"
      auth:
        username: "${INGEST_USERNAME}"
        password: "${INGEST_PASSWORD}"
      framing:
        method: newline_delimited
      decoding:
        codec: json

  transforms:
    route_by_path:
      type: route
      inputs: ["ingest"]
      route:
        fastly: '._ingest_path == "/fastly"'
        upsun: 'starts_with(string!(._ingest_path), "/upsun/")'

    parse_fastly:
      type: remap
      inputs: ["route_by_path.fastly"]
      source: |
        .log_source = "fastly"
        ts, err = parse_timestamp(.time, "%Y-%m-%dT%H:%M:%S", timezone: "UTC")
        ."@timestamp" = if err == null { ts } else { now() }
        del(._ingest_path)
        del(.source_type)

    parse_upsun:
      type: remap
      inputs: ["route_by_path.upsun"]
      source: |
        .log_source = "upsun"
        parts = split(string!(._ingest_path), "/")
        # Prefer the project ID Upsun stamps on the event; fall back to the URL path.
        .upsun_project = string(.container.project) ?? parts[2]
        # Upsun sends the event time as epoch nanoseconds.
        ns, err = to_int(.timestamp_ns)
        ."@timestamp" = if err == null { from_unix_timestamp!(ns, unit: "nanoseconds") } else { now() }
        # Nginx lines arrive as "<container> nginx: <log line>".
        # Parsed fields go under .nginx so they never clash with Upsun's own fields.
        m, err = parse_regex(string(.message) ?? "", r'^\S+ nginx: (?P<line>.*)$')
        if err == null {
          line = string(m.line) ?? ""
          a, err = parse_nginx_log(line, "combined")
          if err == null {
            .nginx = a
            .log_file = "access.log"
          } else {
            # Anything else Nginx writes to the journal is its error log.
            # The journal drops Nginx's date prefix, so fall back to a looser pattern.
            .log_file = "error.log"
            e, err = parse_nginx_log(line, "error")
            if err == null {
              .nginx = e
            } else {
              .nginx = {"message": line}
              v, err = parse_regex(line, r'\[(?P<x>[a-z]+)\]')
              if err == null { .nginx.severity = v.x }
              v, err = parse_regex(line, r'client: (?P<x>[^,]+)')
              if err == null { .nginx.client = v.x }
              v, err = parse_regex(line, r'request: "(?P<x>[^"]*)"')
              if err == null { .nginx.request = v.x }
              v, err = parse_regex(line, r'host: "(?P<x>[^"]*)"')
              if err == null { .nginx.host = v.x }
            }
          }
        }
        # Keep field types stable so OpenSearch's mapping never rejects an event.
        if exists(.timestamp) && !is_integer(.timestamp) { del(.timestamp) }
        if exists(.host) && !is_object(.host) { .host_name = del(.host) }
        del(._ingest_path)
        del(.source_type)

  sinks:
    opensearch:
      type: elasticsearch
      inputs: ["parse_fastly", "parse_upsun"]
      endpoints: ["http://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}"]
      api_version: v8                # typeless bulk requests; OpenSearch 2+ rejects _type
      mode: bulk
      bulk:
        index: "logs-{{ log_source }}-%Y.%m.%d"   # Vector 0.57+ needs a literal prefix
      auth:
        strategy: basic
        user: "${OPENSEARCH_USERNAME}"
        password: "${OPENSEARCH_PASSWORD}"
      buffer:
        type: disk
        max_size: 1073741824         # 1 GiB, lives on the data mount
        when_full: block
```

What this does:

* One source, many senders. Upsun gives each app a single port, so one `http_server` source takes all traffic, and the `route` transform splits it by path.
* Per-source parsing. Fastly's `time` field has no time zone, so it's parsed as UTC into `@timestamp`. Upsun events get an `upsun_project` field from Upsun's own `container.project` metadata, which is how you tell projects apart.
* Daily indices. Events land in `logs-fastly-YYYY.MM.DD` and `logs-upsun-YYYY.MM.DD`. Daily indices make retention a matter of deleting old indices (see [Security, retention, and cost](#security-retention-and-cost)).
* Disk buffer. If OpenSearch is slow or restarting, Vector queues up to 1 GiB on disk instead of dropping logs.

`parse_upsun` reads the event time from Upsun's `timestamp_ns` field. It also spots Nginx lines, which arrive as `<container> nginx: <log line>`, and parses them into an `nginx` object (`nginx.client`, `nginx.path`, `nginx.status`, ...) with `log_file` set to `access.log` or `error.log`. Keeping them in their own object stops them clashing with Upsun's `host` and `timestamp` fields, which OpenSearch would otherwise reject as a mapping conflict.

Vector 0.57 and later. Two defaults changed in 0.57. First, `${VAR}` references in the config are no longer filled in unless Vector starts with `--dangerously-allow-env-var-interpolation`, which the start command in Step 2 includes. Every credential and port here comes from Upsun environment variables, so leave it on. Second, index names and S3 key prefixes built from templates must start with fixed text, which is why they begin with `logs-` and `logs/`.

## Step 4: Deploy and size resources

Commit and push. The first deploy builds both apps and provisions OpenSearch:

```sh theme={null}
  git add . && git commit -m "Log ingestion: Vector + OpenSearch"
  upsun push
```

Then give OpenSearch enough memory and disk. The defaults are too small for log workloads:

```sh theme={null}
  upsun resources:set \
    --size vector:0.5,dashboards:1,opensearch:1 \
    --disk opensearch:20480,vector:2048
```

Give Vector disk for its buffer. The Vector app's `data` mount defaults to 512 MB, but the OpenSearch sink reserves a 1 GiB disk buffer. Vector refuses to start if the buffers don't fit (`Mountpoint '/app/data' has total capacity of … bytes`), which shows up as a `502` on the ingestion URL. 2 GB covers the default config. If you add the optional archive sink, which has its own 1 GiB buffer, use `vector:3072`.

Use this table as a starting point, then adjust from real usage with `upsun metrics`:

| Daily raw log volume | Vector CPU | OpenSearch CPU | OpenSearch disk (14 days hot) |
| :- | :- | :- | :- |
| Under 1 GB | 0.5 | 1 | 20 GB |
| 1 to 10 GB | 1 | 2 | 200 GB |
| 10 to 50 GB | 2 | 4 | 1 TB |

Above about 50 GB a day, shorten hot retention, add the optional object storage archive (pattern A), or talk to your account team about a larger setup.

Match the Dashboards version. Dashboards refuses to connect to a different OpenSearch version. Check what Upsun runs, then set `OSD_VERSION` in `.upsun/config.yaml` to the same number and push again:

```sh theme={null}
  upsun ssh -A vector -- \
    'curl -s -u "$OPENSEARCH_USERNAME:$OPENSEARCH_PASSWORD" "http://$OPENSEARCH_HOST:$OPENSEARCH_PORT"' \
    | grep '"number"'
```

Add your domain (optional). Vector's route is `https://{default}/` and Dashboards is `https://dashboards.{default}/`. Set your ingestion hostname as the default domain and add a CNAME for each hostname, pointing at the target that `upsun domain:add` prints:

```sh theme={null}
  upsun domain:add logs.example.com
  # Vector:      https://logs.example.com/
  # Dashboards:  https://dashboards.logs.example.com/
```

Or skip the domain. A custom domain isn't required. Run `upsun url --primary` to open the primary route, or `upsun url --pipe` to list the generated URLs, which look like this:

```text theme={null}
  https://main-bvxea6i-abcdefgh1234567.eu-3.platformsh.site/              # Vector
  https://dashboards.main-bvxea6i-abcdefgh1234567.eu-3.platformsh.site/   # Dashboards
```

Use them wherever this guide says `logs.example.com` or `dashboards.logs.example.com`, including the Fastly endpoint and challenge URL. They change only if the environment is renamed or recreated.

### Set the Dashboards password

Caddy checks a username and password stored as project variables. The app hashes the password each time it starts, so you only ever handle the plain value:

```sh theme={null}
  DASH_PASS="$(openssl rand -hex 24)"; echo "$DASH_PASS"   # store this in your password manager

  upsun variable:create --level project --name env:DASHBOARDS_USER \
    --value "admin" --visible-build false --visible-runtime true
  upsun variable:create --level project --name env:DASHBOARDS_PASSWORD \
    --value "$DASH_PASS" --sensitive true --visible-build false --visible-runtime true
```

Creating the variables redeploys the environment. Until both exist, the Dashboards URL returns an error. Then open `https://dashboards.logs.example.com/` and, at the browser's login prompt, enter `admin` and the password from `DASH_PASS`. To change it later, run `upsun variable:update --level project --value "<new password>" env:DASHBOARDS_PASSWORD`.

Upsun's environment-level HTTP access control in the Console isn't used here: it applies to every route, so it would also block Fastly and your log senders.

## Step 5: Verify the pipeline

Prove the whole chain works with a sample event before you connect Fastly or any Upsun project. Come back to parts 3 and 4 once real logs are flowing.

1. Send a test event. This should return `200`:

```sh theme={null}
  curl -i -u 'logshipper:<password>' -H 'Content-Type: application/json' \
    --data '{"time":"2026-01-01T12:00:00","service":{"id":"test"},"response":{"status":200}}' \
    https://logs.example.com/fastly
```

A `401` means the credentials are wrong. A `404` means the path didn't match a route.

**2. Check that indices exist:**

```sh theme={null}
  upsun ssh -A vector -- \
    'curl -s -u "$OPENSEARCH_USERNAME:$OPENSEARCH_PASSWORD" "http://$OPENSEARCH_HOST:$OPENSEARCH_PORT/_cat/indices/logs-*?v"'
```

3. Inspect raw events (optional). To see exactly what Fastly or Upsun sends before you adjust the parsing, add this to the top of `vector.yaml`, push, then tap the stream:

```yaml theme={null}
  api:
    enabled: true
    address: "127.0.0.1:8686"
```

```sh theme={null}
  upsun ssh -A vector -- './bin/vector tap "route_by_path.*" --limit 5'
```

4. Explore in Dashboards. Open `https://dashboards.logs.example.com/app/home` and sign in at the browser prompt. If you see a Dashboards 404 page (`{"statusCode":404,"error":"Not Found"}`), the browser requested the wrong path; open `/app/home` directly. On first load, Dashboards may offer a "New! Enhanced Discover experience" popup. Click **Dismiss**: this guide uses the standard Discover, and those settings aren't needed. On the "Welcome to OpenSearch Dashboards" screen, click **Explore on my own** (Add data only offers sample datasets).

Go to **Dashboards Management > Index patterns**, and create `logs-fastly-*` and `logs-upsun-*`, both with `@timestamp` as the time field. Then try these in **Discover**:

| Question | Query (DQL) |
| :- | :- |
| Origin or edge errors | `response.status >= 500` |
| Cache misses at the edge only | `response.state:MISS* and server.is_shield:false` |
| Traffic from one client | `client.ip:"203.0.113.10"` |
| One suspicious TLS fingerprint | `client.ja4_fingerprint:"<ja4>"` |
| Logs from one Upsun project | `upsun_project:"<project-id>"` |

## Step 6: Stream Fastly logs

Fastly sends logs to an HTTPS endpoint only after the endpoint proves it expects them, then posts batches of newline-delimited JSON.

### Publish the ownership challenge

Fastly fetches `https://logs.example.com/.well-known/fastly/logging/challenge` and looks for the SHA-256 hash of your service ID. Put one hash per line, one line per Fastly service:

```sh theme={null}
  mkdir -p vector/public/.well-known/fastly/logging
  printf '%s' "<fastly-service-id>" | shasum -a 256 | cut -d' ' -f1 \
    >> vector/public/.well-known/fastly/logging/challenge
  cat vector/public/.well-known/fastly/logging/challenge   # one 64-character hex line per service
  git add vector/public && git commit -m "Fastly logging challenge" && upsun push
```

Use `shasum -a 256`, which works on macOS and Linux. `sha256sum` doesn't exist on macOS, and the command above would silently write an empty file. Use the service **ID** (for example `SU1Z0isxPaozGVKXdv0eY`), not the service name.

Don't know your Fastly service ID? Upsun Support can tell you. Until then, put a single `*` in the file, which accepts any Fastly service; Vector's password check still protects the endpoint.

Confirm it's served without auth: `curl https://logs.example.com/.well-known/fastly/logging/challenge`.

### Request the HTTPS logging endpoint

Upsun manages the Fastly service in front of your project, so Upsun Support adds the logging endpoint for you. Once the challenge file is live, open a ticket from the [Upsun Console](https://console.upsun.com) with the text below, filled in with your values.

Generate the header value first:

```sh theme={null}
  printf '%s' 'logshipper:<INGEST_PASSWORD>' | base64 | tr -d '\n'
```

Then use this as the ticket body:

```text theme={null}
  Subject: Add a Fastly HTTPS log streaming endpoint

  Project ID:   <upsun-project-id>
  Environment:  main
  Domain(s):    <www.example.com>

  Please add an HTTPS logging endpoint to the Fastly service for this
  project with the settings below, then activate the new service version.

  Name:                   upsun-log-ingestion
  URL:                    https://logs.example.com/fastly
  Method:                 POST
  Content type:           application/json
  Custom header name:     Authorization
  Custom header value:    Basic <base64 value from the command above>
  JSON log entry format:  Newline delimited
  Log line format:        Blank
  Placement:              Format Version Default
  Log format:             the JSON at the end of this ticket, on one line

  The ownership challenge is published at:
  https://logs.example.com/.well-known/fastly/logging/challenge

  Please also confirm our Fastly service ID.

  Log format:
  <paste the one-line log format below>
```

<Warning>
  Replace `logs.example.com` with your ingestion hostname (or the generated `platformsh.site` URL). The header value is a credential: the ticket is private to your organization, but rotate `INGEST_PASSWORD` if it's ever shared more widely.
</Warning>

This is the log format to paste at the end of the ticket. Keep it on **one line**, because each newline starts a new event. Replace `<your-service-name>` with a readable name for the site:

```text theme={null}
  { "time":"%{begin:%Y-%m-%dT%H:%M:%S}t", "timings":{ "to_first_byte":%{time.to_first_byte}V, "elapsed":%{time.elapsed}V }, "service":{ "id":"%{req.service_id}V", "name":"<your-service-name>" }, "client":{ "ja3_fingerprint":"%{if(fastly.ff.visits_this_service==0, json.escape(tls.client.ja3_md5), "")}V", "ja4_fingerprint":"%{if(fastly.ff.visits_this_service==0, json.escape(tls.client.ja4), "")}V", "ip":"%{json.escape(req.http.Fastly-Client-IP)}V" }, "server":{ "datacenter":"%{json.escape(server.datacenter)}V", "is_shield":%{if(fastly.ff.visits_this_service==0, "false", "true")}V, "node":"%{json.escape(server.identity)}V" }, "request":{ "method":"%{json.escape(req.request)}V", "host":"%{json.escape(req.http.host)}V", "path":"%{json.escape(req.url)}V", "proto":"%{json.escape(req.proto)}V", "header_bytes":%{req.header_bytes_read}V, "body_bytes":%{req.body_bytes_read}V, "user_agent":"%{json.escape(req.http.user-agent)}V", "referer":"%{json.escape(req.http.referer)}V", "origin":"%{json.escape(req.http.origin)}V", "content_type":"%{json.escape(req.http.content-type)}V" }, "response":{ "status":%{resp.status}V, "state":"%{json.escape(fastly_info.state)}V", "header_bytes":%{resp.header_bytes_written}V, "body_bytes":%{resp.body_bytes_written}V, "content_type":"%{json.escape(resp.http.content-type)}V" }, "tls":{ "is_tls":%{if(req.is_ssl, "true", "false")}V, "cipher":"%{if(req.is_ssl, json.escape(tls.client.cipher), "")}V", "protocol":"%{json.escape(tls.client.protocol)}V" } }
```

About the fields:

* `time` is the request start in UTC. Vector copies it into `@timestamp`.
* `service.id` identifies the Fastly service, so many services can share one endpoint and you can filter on it.
* `ja3_fingerprint` and `ja4_fingerprint` are only filled on the first Fastly hop (the edge). On shield nodes they're empty and `server.is_shield` is `true`.

Once Support confirms the new version is active, logs usually appear within a minute. Check with Step 5, part 2. For each additional Fastly service or project, add its challenge hash and open another ticket with the same settings.

## Step 7: Forward logs from Upsun projects

Each source project gets an `httplog` [log forwarding](/docs/observability/logs/forward-logs) integration that posts to its own path on the ingestion endpoint. Use the source project's own ID in the path. Upsun also stamps the project ID on every event, and Vector stores it as `upsun_project`.

Forwarding sends everything that goes through the system journal of each app container. On Upsun Cloud that includes Nginx's access and error logs, so one integration per project covers the usual `/var/log` files:

| Log | Forwarded by `httplog`? |
| :- | :- |
| `app.log` (stdout and stderr) | Yes |
| `syslog.log` (messages your app sends to syslog) | Yes |
| `access.log` (Nginx) | Yes, as `nginx` journal messages |
| `error.log` (Nginx) | Yes, as `nginx` journal messages |
| `post-start.log` and files your app writes itself | No: log to stdout or syslog instead |
| `dns.log` | No |

This works the same for PHP (Drupal, WordPress, Magento), Node.js, Python, and decoupled projects. Nothing needs to change in the source app's code or `.upsun/config.yaml`.

### One project

```sh theme={null}
  SRC="<source-project-id>"
  AUTH="Basic $(printf '%s' 'logshipper:<password>' | base64 | tr -d '\n')"

  upsun integration:add --project "$SRC" --type httplog \
    --url "https://logs.example.com/upsun/$SRC" \
    --header "Authorization: $AUTH" \
    --header "Content-Type: application/json"

  upsun redeploy --project "$SRC" --environment main
```

Forwarding starts after the redeploy. Repeat the redeploy for any other environment you want to ship.

### Many projects

List the project IDs in a file, one per line, and loop:

```sh theme={null}
  while read -r SRC; do
    upsun integration:add --project "$SRC" --type httplog --yes \
      --url "https://logs.example.com/upsun/$SRC" \
      --header "Authorization: $AUTH" \
      --header "Content-Type: application/json"
    upsun redeploy --project "$SRC" --environment main --yes
  done < projects.txt
```

Plan requirements. You need project admin access on each source project, and forwarding only works on Grid projects (Dedicated uses rsyslog instead). Real-time log forwarding is billed separately; on Upsun Fixed it's part of the Observability Suite. Check with your account team if `integration:add` refuses the `httplog` type.

**Why not syslog?** Upsun also supports syslog forwarding, but syslog needs a raw TCP or TLS port. Upsun apps are only reachable over HTTP(S) through the router, so this project accepts `httplog` only.

### What arrives and how to find it

Each forwarded event carries Upsun's metadata. The useful fields are:

| Field | Example | Meaning |
| :- | :- | :- |
| `upsun_project` | `abcdefgh1234567` | Source project, from Upsun's `container.project` (URL path as fallback) |
| `container.service` | `drupal`, `frontend` | App that wrote the line |
| `container.environment` | `main` | Environment |
| `component.id.name` | `nginx`, `php-fpm` | Process that wrote the line |
| `message` | `drupal.0 nginx: 203.0.113.10 - - [...] "GET / HTTP/1.1" 200 ...` | Raw line |

Nginx lines are parsed by `parse_upsun` (Step 3). Access lines get `log_file: "access.log"` plus `nginx.client`, `nginx.method`, `nginx.path`, `nginx.protocol`, `nginx.status`, `nginx.size`, `nginx.referer` and `nginx.agent`. Error lines get `log_file: "error.log"` plus `nginx.message`, and `nginx.severity`, `nginx.client`, `nginx.request` and `nginx.host` when the line includes them.

After the first events arrive, refresh the index pattern so the new fields show up: go to **Dashboards Management → Index patterns**, open `logs-upsun-*` and click the refresh icon. Then try these in **Discover**:

| To see | Query |
| :- | :- |
| Access log for one project | `upsun_project:"<project-id>" and log_file:"access.log"` |
| One app in a decoupled project | `container.service:"frontend" and log_file:"access.log"` |
| Server errors | `log_file:"access.log" and nginx.status >= 500` |
| Not found | `log_file:"access.log" and nginx.status:404` |
| Nginx errors | `log_file:"error.log"` |
| App output (stdout/stderr) | `upsun_project:"<project-id>" and not component.id.name:"nginx"` |

Add `container.service`, `nginx.client`, `nginx.method`, `nginx.path`, `nginx.status` and `nginx.agent` as columns, then **Save** the search for one-click access.

**Requests served from Fastly's cache never reach the app**, so they aren't in the app's access log. Use `logs-fastly-*` for the full edge view.

### Other file logs

`httplog` doesn't forward logs your app writes to its own files, or `post-start.log`. Send those messages to stdout or syslog instead: Drupal's syslog module, WordPress's `error_log()` with `error_log = syslog`, or Python's and Node.js's standard loggers writing to stdout. They then arrive through the same integration with no extra setup.

## Security, retention, and cost

### Security

<Warning>
  * Send logs over `https://`. The router terminates TLS; the examples use HTTP on the private network for OpenSearch and object storage.
  * Rotate the ingest password. Update `env:INGEST_PASSWORD` (this redeploys), then update the Fastly header and re-run `integration:update` on each source project. Expect a short gap in logs.
  * One Dashboards login. Anyone with the Dashboards password can see every log, so share it only with people who need it. To give someone separate credentials, add another `user hash` line to the Caddyfile's `basic_auth` block.
  * The challenge file is public on purpose. It holds only hashes of service IDs.
</Warning>

**Three sets of credentials:**

| Credential | Where it's set | Who uses it |
| :- | :- | :- |
| `DASHBOARDS_USER` / `DASHBOARDS_PASSWORD` | Project variables (Step 4) | People opening Dashboards, checked by Caddy |
| `INGEST_USERNAME` / `INGEST_PASSWORD` | Project variables (Step 3) | Fastly and source Upsun projects, checked by Vector |
| `OPENSEARCH_USERNAME` / `OPENSEARCH_PASSWORD` | Generated by Upsun when `authentication.enabled: true` | Vector's OpenSearch sink, Dashboards' server connection, and every `curl` in this guide (Steps 4 and 5, retention cron) |

OpenSearch has no authentication by default on Upsun, so keep `authentication.enabled: true` even though it's only reachable internally.

### Retention

Add this [cron job](/docs/configure-apps/image-properties/crons) to the `vector` app in `.upsun/config.yaml`. It deletes OpenSearch indices older than 14 days:

```yaml theme={null}
      crons:
        opensearch-retention:
          spec: "15 3 * * *"
          commands:
            start: |
              OS="http://$OPENSEARCH_HOST:$OPENSEARCH_PORT"
              CUTOFF=$(date -u -d "14 days ago" +%Y%m%d)
              for IDX in $(curl -s -u "$OPENSEARCH_USERNAME:$OPENSEARCH_PASSWORD" "$OS/_cat/indices/logs-*?h=index"); do
                DAY=$(echo "${IDX##*-}" | tr -d .)
                [ "$DAY" -lt "$CUTOFF" ] && curl -s -X DELETE -u "$OPENSEARCH_USERNAME:$OPENSEARCH_PASSWORD" "$OS/$IDX"
              done
```

Upsun's OpenSearch also supports the Index State Management (ISM) plugin. If you enable it on the service, an ISM delete policy on `logs-*` can replace the `opensearch-retention` cron.

### Cost

* OpenSearch disk is the biggest line. Shorten hot retention before adding disk.
* Drop fields you never query. A `del(.field)` in the `parse_fastly` transform shrinks every document.
* Sample noisy traffic. Vector's `sample` transform can keep, say, 1 in 10 successful static-asset hits while keeping every error.
* Object storage is billed on allocation. 512 GB is the minimum, so compare the allocated storage cost with your expected OpenSearch retention before enabling an archive.
* Avoid long-lived preview environments. A branch of this project clones OpenSearch and runs a second ingest stack. Point senders only at `main`.

## Optional: Archive logs to object storage

Upsun [object storage](/docs/manage-resources/object-storage) is an S3-compatible bucket attached to one app. Compressed JSON in a bucket costs far less per GB than OpenSearch disk, so it's a good home for logs you must keep but rarely search. Choose one of two patterns:

| Pattern | OpenSearch keeps | Object storage keeps | Best for |
| :- | :- | :- | :- |
| A. Hot + archive | 7 to 14 days | Everything, gzip-compressed | Teams that search recent logs and need long retention |
| B. Archive only | Nothing (no OpenSearch or Dashboards) | Everything | Compliance or audit retention where search is rare |

### Enable the bucket

Object storage is sized in 512 GB steps (minimum 512 GB, maximum 10 TB) and billed on allocated size, not usage. Attach it to the Vector app:

```sh theme={null}
  upsun resources:set --object-storage vector:524288   # 512 GB, in MB
```

The bucket is named after the app (`vector`). Upsun injects `OBJECT_STORAGE_HOST`, `OBJECT_STORAGE_PORT`, `OBJECT_STORAGE_ACCESS_KEY`, `OBJECT_STORAGE_SECRET_KEY` and `OBJECT_STORAGE_REGION` into the Vector app. The endpoint is internal only, so no one outside the project can reach it.

### Add the archive sink

Raise the Vector app's disk for the second buffer (`upsun resources:set --disk vector:3072`), then append this sink to `vector/vector.yaml`:

```yaml theme={null}
    archive:
      type: aws_s3
      inputs: ["parse_fastly", "parse_upsun"]
      endpoint: "http://${OBJECT_STORAGE_HOST}:${OBJECT_STORAGE_PORT}"
      region: "${OBJECT_STORAGE_REGION}"
      bucket: "vector"
      force_path_style: true         # Upsun requires path-style addressing
      auth:
        access_key_id: "${OBJECT_STORAGE_ACCESS_KEY}"
        secret_access_key: "${OBJECT_STORAGE_SECRET_KEY}"
      key_prefix: "logs/{{ log_source }}/%Y/%m/%d/"
      compression: gzip
      encoding:
        codec: json
      framing:
        method: newline_delimited
      batch:
        max_bytes: 52428800          # 50 MB per object before compression
        timeout_secs: 300            # or every 5 minutes
      buffer:
        type: disk
        max_size: 1073741824
        when_full: block
```

For **pattern B**, also delete the `opensearch` sink, the `dashboards` app, the `opensearch` service, the `dashboards.{default}` route and the `opensearch` relationship from the Vector app.

### Things to know

* No lifecycle rules. Upsun object storage doesn't support lifecycle policies, versioning, object lock, server-side encryption or bucket policies. Expire old objects with the cron job below.
* Deletion is permanent. Setting the size to `0` deletes the bucket and everything in it on the next deployment.
* Reading archives. Use `upsun ssh -A vector` and the AWS CLI with `--endpoint-url "http://$OBJECT_STORAGE_HOST:$OBJECT_STORAGE_PORT"` to list or download files. To search old logs, download a day's prefix and load it into your tool of choice, or re-ingest it into OpenSearch.

### Expire old archives

Add the AWS CLI and a second cron job to the `vector` app, next to `opensearch-retention`. This one deletes the archive day that just aged out, here 400 days:

```yaml theme={null}
      dependencies:
        python3:
          awscli: "*"
      crons:
        archive-retention:
          spec: "45 3 * * *"
          commands:
            start: |
              export AWS_ACCESS_KEY_ID="$OBJECT_STORAGE_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$OBJECT_STORAGE_SECRET_KEY" AWS_DEFAULT_REGION="$OBJECT_STORAGE_REGION"
              OLD=$(date -u -d "400 days ago" +%Y/%m/%d)
              for SRC in fastly upsun; do
                aws s3 rm "s3://vector/logs/$SRC/$OLD/" --recursive --endpoint-url "http://$OBJECT_STORAGE_HOST:$OBJECT_STORAGE_PORT"
              done
```

The job removes one day per run. If it misses a day, delete that prefix by hand.

## Troubleshooting

Start with the Vector app log: `upsun log -A vector app --tail`.

| Symptom | Likely cause | Fix |
| :- | :- | :- |
| Fastly won't save the endpoint: "unable to validate" | Challenge file missing or hash wrong | `curl` the challenge URL; regenerate the hash using the command in Step 6 |
| Senders get `401` | Wrong or stale basic auth | Regenerate the base64 value from the current `INGEST_USERNAME` and `INGEST_PASSWORD` |
| Vector log: `failed to parse JSON` | Log format split across lines, or JSON entry format isn't newline delimited | Put the Fastly format on one line and set JSON log entry format to Newline delimited |
| Build fails downloading Vector or Dashboards | Version number doesn't exist | Check the release pages and fix `VECTOR_VERSION` or `OSD_VERSION` |
| Dashboards shows "incompatible version" | `OSD_VERSION` differs from OpenSearch | Repeat the version check in Step 4 |
| Events in the wrong day's index or `@timestamp` = ingest time | Timestamp field missing or different format | Tap a raw event (Step 5) and check the timestamp lines in `parse_fastly` or `parse_upsun` |
| Vector log: `429` or bulk rejections from OpenSearch | OpenSearch undersized | Raise OpenSearch CPU with `upsun resources:set`; the disk buffer holds logs meanwhile |
| OpenSearch disk full, indices read-only | Retention too long for the disk | Delete old indices, lower the retention cutoff, or add disk |
| Archive sink errors mentioning the bucket or host | Object storage not enabled, or virtual-hosted addressing | Run `resources:set --object-storage` and keep `force_path_style: true` |
| No Upsun logs arriving | Integration added but project not redeployed | `upsun redeploy` the source environment |
| Vector log: `mapper_parsing_exception` | A field changed type within today's index | Keep the type guards at the end of `parse_upsun`; affected events are dropped until the next day's index |

**Expected and harmless:**

* **`Transform "route_by_path._unmatched" has no consumers`** at Vector startup. Requests to paths other than `/fastly` or `/upsun/<id>` are dropped on purpose.
* A new index takes a few seconds to appear. Vector batches events before sending them. If `_cat/indices` is empty right after the first test event, wait 10 to 20 seconds and check again.
* `vector tap` says the API isn't reachable. The `api:` block must start at column 0 in `vector.yaml`, level with `data_dir:`, and you must push after adding it.

Dashboards shows a generic `Error ... at Fetch` and then `502`s. Check `upsun log -A dashboards app`. `JavaScript heap out of memory` means Dashboards ran out of memory, usually on a Discover search over a long time range. Upsun supplies the Node heap setting through `NODE_OPTIONS`, and Supervisor restarts Dashboards if it exits. If it keeps happening, raise the app's size, for example `upsun resources:set --size dashboards:2`, and keep Discover time ranges short.

Upsun logs stop after the ingestion endpoint was down. If Vector was stopped or failing for a while, for example during a bad config push, source projects can stop forwarding even after it recovers. Today's `logs-upsun-*` `docs.count` stays flat. Redeploy each source project to restart forwarding: `upsun redeploy -p <source-project-id> -e main`.

### Sources

* [Upsun: Forward logs](/docs/observability/logs/forward-logs)
* [Upsun: OpenSearch service](/docs/add-services/opensearch)
* [Upsun: Object storage](/docs/manage-resources/object-storage)
* [Upsun: Crons](/docs/configure-apps/image-properties/crons)
* [Vector documentation](https://vector.dev/docs/)
* [Vector 0.57 upgrade guide](https://vector.dev/highlights/2026-07-14-0-57-0-upgrade-guide/)


This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.