remove stuff that no longer belongs here
This commit is contained in:
parent
71c40244d3
commit
73a7ed7480
|
@ -1,18 +0,0 @@
|
||||||
"""Ad-hoc script which tags uploads with a sequential number."""
|
|
||||||
import sys
|
|
||||||
|
|
||||||
uploading = False
|
|
||||||
upload_no = 0
|
|
||||||
for line in sys.stdin:
|
|
||||||
if 'Handling file upload' in line:
|
|
||||||
upload_no += 1
|
|
||||||
uploading = True
|
|
||||||
|
|
||||||
if uploading:
|
|
||||||
line = line.strip()
|
|
||||||
parts = line.rsplit(' ', maxsplit=1)
|
|
||||||
line = ' '.join([parts[0], f'upload={upload_no}', parts[1]])
|
|
||||||
print(line)
|
|
||||||
|
|
||||||
if 'Uploaded file' in line:
|
|
||||||
uploading = False
|
|
|
@ -1,208 +0,0 @@
|
||||||
---
|
|
||||||
title: "R Notebook"
|
|
||||||
output: html_notebook
|
|
||||||
---
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
library(tidyverse)
|
|
||||||
library(lubridate)
|
|
||||||
```
|
|
||||||
|
|
||||||
# Node Crashing on Upload
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
uploads <- read_csv('./codex-continuous-tests-0codex3-5-77bdb95dc7-j7f46_codex3-5-uploads.csv')
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
durations <- uploads |>
|
|
||||||
arrange(count) |>
|
|
||||||
group_by(upload) |>
|
|
||||||
summarise(
|
|
||||||
start = timestamp[1],
|
|
||||||
end = timestamp[n()],
|
|
||||||
) |>
|
|
||||||
mutate(duration = end - start)
|
|
||||||
```
|
|
||||||
|
|
||||||
How long are uploads taking?
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
ggplot(durations, aes(x = upload, y = duration)) +
|
|
||||||
geom_point() +
|
|
||||||
geom_line() +
|
|
||||||
ylab('upload duration') +
|
|
||||||
xlab('upload number') +
|
|
||||||
theme_minimal()
|
|
||||||
```
|
|
||||||
Are all uploads completing?
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
uploads |>
|
|
||||||
filter(message == 'Got data from stream') |>
|
|
||||||
group_by(upload) |>
|
|
||||||
count(name = 'blocks')
|
|
||||||
```
|
|
||||||
|
|
||||||
Does the end of the upload coincide with the last chunk that gets stored?
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
uploads |>
|
|
||||||
filter(grepl('Got data from stream', message)) |>
|
|
||||||
group_by(upload) |>
|
|
||||||
summarise(
|
|
||||||
last_store = max(timestamp)
|
|
||||||
) |>
|
|
||||||
inner_join(durations, by='upload')
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
durations
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
uploads |> filter(grepl('Exception', message)) |> group_by(message) |> count() |> arrange(n)
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
uploads |> filter(upload == 18) |> group_by(message) |> count() |> arrange(n)
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
uploads |> filter(upload == 17) |> group_by(message) |> count() |> arrange(n)
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
messages <- uploads |> group_by(message) |> count() |> filter(n > 100) |> pull(message)
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
```{r fig.height = 10}
|
|
||||||
uploads |> filter(message %in% messages) |> group_by(upload, message) |> count() %>% {
|
|
||||||
ggplot(.) +
|
|
||||||
geom_point(aes(x = message, y = n, color = as.factor(upload))) + theme_minimal() + theme(axis.text.x = element_text(angle = 45, hjust=1)) +
|
|
||||||
ylab('count') +
|
|
||||||
scale_color_manual(values=c('18'='red'))
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
interlog_intervals <- uploads |>
|
|
||||||
group_by(upload) |>
|
|
||||||
arrange(timestamp) |>
|
|
||||||
mutate(log_interval = as.numeric(timestamp - lag(timestamp))) |>
|
|
||||||
ungroup()
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
interlog_intervals |>
|
|
||||||
group_by(upload) |>
|
|
||||||
summarise(
|
|
||||||
mean_li = mean(log_interval, na.rm=TRUE),
|
|
||||||
median_li = median(log_interval, na.rm=TRUE),
|
|
||||||
max_li = max(log_interval, na.rm=TRUE),
|
|
||||||
) |>
|
|
||||||
pivot_longer(-upload) %>% {
|
|
||||||
ggplot(.) +
|
|
||||||
geom_line(aes(x = upload, y = value, col = name)) +
|
|
||||||
scale_y_log10() +
|
|
||||||
theme_minimal() +
|
|
||||||
ylab('duration (logscale, seconds)')
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
interlog_intervals |> group_by(upload) |> count() |> arrange(desc(n))
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
```{r fig.height=5}
|
|
||||||
interlog_intervals |>
|
|
||||||
group_by(upload) |>
|
|
||||||
arrange(log_interval) |>
|
|
||||||
mutate(rank = seq_along(log_interval)) |> ungroup() %>% {
|
|
||||||
ggplot(.) +
|
|
||||||
geom_point(aes(x = rank, y = log_interval, col = as.factor(upload))) +
|
|
||||||
theme_minimal() +
|
|
||||||
xlab('rank') +
|
|
||||||
ylab('time between two consecutive log messages') +
|
|
||||||
guides(col = guide_legend(title = 'upload #'))
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
ggplot(
|
|
||||||
interlog_intervals |>
|
|
||||||
filter(upload == 18
|
|
||||||
) |>
|
|
||||||
mutate(bucket = floor_date(timestamp, unit = '5 seconds')) |>
|
|
||||||
group_by(bucket) |>
|
|
||||||
mutate(
|
|
||||||
mean_interval = mean(log_interval),
|
|
||||||
p_70 = quantile(log_interval[-1], probs = c(0.95))
|
|
||||||
) |>
|
|
||||||
ungroup()
|
|
||||||
) +
|
|
||||||
geom_point(aes(x = timestamp, y = log_interval)) +
|
|
||||||
geom_line(aes(x = bucket, y = mean_interval), col = 'red', lwd = 2) +
|
|
||||||
geom_line(aes(x = bucket, y = p_70), col = 'orange', lwd = 2) +
|
|
||||||
theme_minimal()
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
# Whole-Cluster
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
cluster_uploads <- read_csv('../data/20/pods/uploads/all_uploads.csv') |> filter(source != 'source')
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
cluster_upload_durations <- cluster_uploads |> group_by(source, upload) |> arrange(timestamp) |> summarise(duration = as.numeric(timestamp[n()] - timestamp[1]))
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r fig.width=12}
|
|
||||||
ggplot(cluster_upload_durations) +
|
|
||||||
geom_line(aes(x = upload, y = duration, col = source)) +
|
|
||||||
theme_minimal() +
|
|
||||||
facet_wrap(. ~ source) +
|
|
||||||
guides(color = FALSE)
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
cluster_interlog_intervals <- cluster_uploads |>
|
|
||||||
group_by(source, upload) |>
|
|
||||||
arrange(timestamp) |>
|
|
||||||
mutate(log_interval = as.numeric(timestamp - lag(timestamp))) |>
|
|
||||||
ungroup()
|
|
||||||
```
|
|
||||||
|
|
||||||
```{r fig.width=10}
|
|
||||||
cluster_interlog_intervals |>
|
|
||||||
group_by(source, upload) |>
|
|
||||||
summarise(
|
|
||||||
mean_li = mean(log_interval, na.rm=TRUE),
|
|
||||||
median_li = median(log_interval, na.rm=TRUE),
|
|
||||||
max_li = max(log_interval, na.rm=TRUE),
|
|
||||||
) |>
|
|
||||||
pivot_longer(-c(source, upload)) %>% {
|
|
||||||
ggplot(.) +
|
|
||||||
geom_line(aes(x = upload, y = value, col = name)) +
|
|
||||||
scale_y_log10() +
|
|
||||||
theme_minimal() +
|
|
||||||
ylab('interval between log messages (logscale, seconds)') +
|
|
||||||
facet_wrap(. ~ source)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
```{r}
|
|
||||||
ggplot(cluster_interlog_intervals) +
|
|
||||||
geom_line(aes(x = upload, y = duration, col = source)) +
|
|
||||||
theme_minimal() +
|
|
||||||
facet_wrap(. ~ source) +
|
|
||||||
guides(color = FALSE)
|
|
||||||
```
|
|
||||||
|
|
|
@ -1,13 +0,0 @@
|
||||||
Version: 1.0
|
|
||||||
|
|
||||||
RestoreWorkspace: Default
|
|
||||||
SaveWorkspace: Default
|
|
||||||
AlwaysSaveHistory: Default
|
|
||||||
|
|
||||||
EnableCodeIndexing: Yes
|
|
||||||
UseSpacesForTab: Yes
|
|
||||||
NumSpacesForTab: 2
|
|
||||||
Encoding: UTF-8
|
|
||||||
|
|
||||||
RnwWeave: Sweave
|
|
||||||
LaTeX: pdfLaTeX
|
|
|
@ -1,8 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
# Concatenates CSV files that have identical headers by removing the header from all but the first file. This is
|
|
||||||
# meant to be used after a call to `cat`; e.g., cat csv1.csv csv2.csv | lscsv-concat.sh
|
|
||||||
set -e
|
|
||||||
|
|
||||||
header=$(head -n 1)
|
|
||||||
echo "$header"
|
|
||||||
grep "$header" -Fv
|
|
|
@ -1,29 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
# Given a namespace and a base folder containing the runner logs for continuous tests, creates
|
|
||||||
# a storage area (folder) and:
|
|
||||||
#
|
|
||||||
# 1. pulls pod logs into storage_area/pods
|
|
||||||
# 2. copies runner logs to storage_area/runner
|
|
||||||
#
|
|
||||||
# Make sure you delete the original runner logs once this is done, as otherwise they might get copied into more
|
|
||||||
# than one storage area.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
namespace=${1}
|
|
||||||
runner_log_source=${2}
|
|
||||||
|
|
||||||
if [ -z "$namespace" ] || [ -z "$runner_log_source" ]; then
|
|
||||||
echo "Usage: bin/process_logs.sh <namespace> <runner_logs>"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
run_id=$(date +'%Y-%m-%d-%H%M%S')
|
|
||||||
logs="data/logs/$run_id"
|
|
||||||
pod_logs="$logs/pods"
|
|
||||||
runner_logs="$logs/runner"
|
|
||||||
|
|
||||||
mkdir -p "$pod_logs"
|
|
||||||
bash ./bin/pull-pod-logs.sh "$namespace" "$pod_logs"
|
|
||||||
|
|
||||||
mkdir -p "$runner_logs"
|
|
||||||
cp "$runner_log_source"/* "$runner_logs/"
|
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
namespace=${1:-"codex-continuous-tests"}
|
|
||||||
output_folder=${2:-./}
|
|
||||||
|
|
||||||
# List all pods in the namespace
|
|
||||||
pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}')
|
|
||||||
|
|
||||||
if [ -z "$pods" ]; then
|
|
||||||
echo "No pods found in namespace $namespace."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
for pod in $pods; do
|
|
||||||
echo "Fetching logs for $pod..."
|
|
||||||
|
|
||||||
# Handle pods with multiple containers
|
|
||||||
containers=$(kubectl get pod "$pod" -n "$namespace" -o jsonpath='{.spec.containers[*].name}')
|
|
||||||
for container in $containers; do
|
|
||||||
if [ "$container" == "$pod" ]; then
|
|
||||||
# If there's only one container, name the log file after the pod
|
|
||||||
kubectl logs "$pod" -n "$namespace" > "${output_folder}/${pod}.log"
|
|
||||||
else
|
|
||||||
# If there are multiple containers, name the log file after the pod and container
|
|
||||||
kubectl logs "$pod" -c "$container" -n "$namespace" > "${output_folder}/${pod}_${container}.log"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "Done fetching logs."
|
|
|
@ -1,4 +0,0 @@
|
||||||
# Snippets
|
|
||||||
|
|
||||||
Ad hoc snippets which reshape data for one-off analysis, not worth the trouble of making into scripts.
|
|
||||||
|
|
|
@ -1,26 +0,0 @@
|
||||||
set -e
|
|
||||||
|
|
||||||
base_folder=${1:-"./data/20"}
|
|
||||||
mkdir -p "${base_folder}/pods/uploads"
|
|
||||||
|
|
||||||
# tags uploads with id
|
|
||||||
for i in "${base_folder}"/pods/codex-continuous-tests-0codex*; do
|
|
||||||
python -m adhoc.identify_uploads < "$i" > "${i%/*}/uploads/${i##*/}"
|
|
||||||
done
|
|
||||||
|
|
||||||
# transforms raw logs into single CSV
|
|
||||||
for i in "${base_folder}"/pods/uploads/codex-continuous-tests-0codex*; do
|
|
||||||
python -m logtools.cli.to_csv < "$i" \
|
|
||||||
--extract-fields upload \
|
|
||||||
--constant-column \
|
|
||||||
source=${${i##*/}%.*} >> "${base_folder}"/pods/uploads/all_uploads.csv.temp
|
|
||||||
done
|
|
||||||
|
|
||||||
./bin/csv-concat.sh < "${base_folder}"/pods/uploads/all_uploads.csv.temp > "${base_folder}"/pods/uploads/all_uploads.csv
|
|
||||||
rm "${base_folder}"/pods/uploads/all_uploads.csv.temp
|
|
||||||
|
|
||||||
# extracts debug endpoint data and looks into wantlist sizes
|
|
||||||
grep -h 'Before upload\|After download' "${base_folder}"/runner/*.log | \
|
|
||||||
sed -E 's/\[(.{28})\] <([A-Z]+[0-9]+)> (Before upload|After download): (.*)$/\4/p' > "${base_folder}"/runner/merged.jsonl
|
|
||||||
|
|
||||||
jq '.pendingBlocks' < "${base_folder}"/runner/merged.jsonl | uniq # should print 0
|
|
Loading…
Reference in New Issue