Skip to content

Commit 5ab5db0

Browse files
astefanuttiastefanutti
committed
test: Upload job logs
1 parent 02ee9e4 commit 5ab5db0

File tree

11 files changed

+90
-19
lines changed

11 files changed

+90
-19
lines changed

‎.github/workflows/e2e_tests.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,9 @@ jobs:
169169
export CODEFLARE_TEST_TIMEOUT_MEDIUM=3m
170170
export CODEFLARE_TEST_TIMEOUT_LONG=8m
171171
172+
export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
173+
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
174+
172175
make test-e2e
173176
174177
- name: Print CodeFlare operator logs
@@ -188,3 +191,12 @@ jobs:
188191
run: |
189192
echo "Printing KubeRay operator logs"
190193
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay
194+
195+
- name: Upload logs
196+
uses: actions/upload-artifact@v3
197+
if: always() && steps.deploy.outcome == 'success'
198+
with:
199+
name: logs
200+
retention-days: 10
201+
path: |
202+
${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log

‎test/e2e/mnist_pytorch_mcad_job_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
151151
WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
152152
))
153153

154-
// Print the job logs
155-
PrintJobLogs(test, job.Namespace, job.Name)
154+
// Retrieve the job logs
155+
WriteJobLogs(test, job.Namespace, job.Name)
156156

157157
// Assert the job has completed successfully
158158
test.Expect(GetJob(test, job.Namespace, job.Name)).

‎test/e2e/mnist_raycluster_sdk_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,8 @@ func TestMNISTRayClusterSDK(t *testing.T) {
195195
WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
196196
))
197197

198-
// Print the job logs
199-
PrintJobLogs(test, job.Namespace, job.Name)
198+
// Retrieve the job logs
199+
WriteJobLogs(test, job.Namespace, job.Name)
200200

201201
// Assert the job has completed successfully
202202
test.Expect(GetJob(test, job.Namespace, job.Name)).

‎test/e2e/mnist_rayjob_mcad_raycluster_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,8 +254,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) {
254254
test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong).
255255
Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal)))
256256

257-
test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name)
258-
test.T().Log(GetRayJobLogs(test, rayJob.Namespace, rayJob.Name))
257+
test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name)
258+
WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name))
259259

260260
// Assert the Ray job has completed successfully
261261
test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)).

‎test/e2e/support.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
var files embed.FS
2929

3030
func ReadFile(t support.Test, fileName string) []byte {
31+
t.T().Helper()
3132
file, err := files.ReadFile(fileName)
3233
t.Expect(err).NotTo(gomega.HaveOccurred())
3334
return file

‎test/support/batch.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func GetJob(t Test, namespace, name string) *batchv1.Job {
3838
return Job(t, namespace, name)(t)
3939
}
4040

41-
func PrintJobLogs(t Test, namespace, name string) {
41+
func WriteJobLogs(t Test, namespace, name string) {
4242
t.T().Helper()
4343

4444
job := GetJob(t, namespace, name)
@@ -51,8 +51,8 @@ func PrintJobLogs(t Test, namespace, name string) {
5151
t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
5252
} else {
5353
for i, pod := range pods {
54-
t.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name)
55-
t.T().Log(GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
54+
t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name)
55+
WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
5656
}
5757
}
5858
}

‎test/support/codeflare.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,18 @@ import (
2020
"os"
2121
)
2222

23-
// The environment variables hereafter can be used to change the components
24-
// used for testing.
2523
const (
24+
// The environment variables hereafter can be used to change the components
25+
// used for testing.
26+
2627
CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION"
2728
CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION"
2829
CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE"
2930
CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE"
31+
32+
// The testing output directory, to write output files into.
33+
34+
CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR"
3035
)
3136

3237
func GetCodeFlareSDKVersion() string {

‎test/support/core.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod
4343
return pods.Items
4444
}
4545

46-
func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string {
46+
func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte {
4747
t.T().Helper()
4848
stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx())
4949
t.Expect(err).NotTo(gomega.HaveOccurred())
@@ -55,5 +55,5 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string {
5555
bytes, err := io.ReadAll(stream)
5656
t.Expect(err).NotTo(gomega.HaveOccurred())
5757

58-
return string(bytes)
58+
return bytes
5959
}

‎test/support/ray.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus {
4545
return job.Status.JobStatus
4646
}
4747

48-
func GetRayJobLogs(t Test, namespace, name string) string {
48+
func GetRayJobLogs(t Test, namespace, name string) []byte {
4949
t.T().Helper()
5050

5151
job := GetRayJob(t, namespace, name)
@@ -61,5 +61,5 @@ func GetRayJobLogs(t Test, namespace, name string) string {
6161
t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed())
6262
t.Expect(body).To(gomega.HaveKey("logs"))
6363

64-
return body["logs"]
64+
return []byte(body["logs"])
6565
}

‎test/support/test.go

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ package support
1818

1919
import (
2020
"context"
21+
"os"
22+
"path"
2123
"sync"
2224
"testing"
2325

@@ -30,6 +32,7 @@ type Test interface {
3032
T() *testing.T
3133
Ctx() context.Context
3234
Client() Client
35+
OutputDir() string
3336

3437
gomega.Gomega
3538

@@ -70,9 +73,13 @@ type T struct {
7073
*gomega.WithT
7174
t *testing.T
7275
// nolint: containedctx
73-
ctx context.Context
74-
client Client
75-
once sync.Once
76+
ctx context.Context
77+
client Client
78+
outputDir string
79+
once struct {
80+
client sync.Once
81+
outputDir sync.Once
82+
}
7683
}
7784

7885
func (t *T) T() *testing.T {
@@ -84,7 +91,8 @@ func (t *T) Ctx() context.Context {
8491
}
8592

8693
func (t *T) Client() Client {
87-
t.once.Do(func() {
94+
t.T().Helper()
95+
t.once.client.Do(func() {
8896
c, err := newTestClient()
8997
if err != nil {
9098
t.T().Fatalf("Error creating client: %v", err)
@@ -94,6 +102,31 @@ func (t *T) Client() Client {
94102
return t.client
95103
}
96104

105+
func (t *T) OutputDir() string {
106+
t.T().Helper()
107+
t.once.outputDir.Do(func() {
108+
if parent, ok := os.LookupEnv(CodeFlareTestOutputDir); ok {
109+
if !path.IsAbs(parent) {
110+
if cwd, err := os.Getwd(); err == nil {
111+
// best effort to output the parent absolute path
112+
parent = path.Join(cwd, parent)
113+
}
114+
}
115+
t.T().Logf("Creating output directory in parent directory: %s", parent)
116+
dir, err := os.MkdirTemp(parent, t.T().Name())
117+
if err != nil {
118+
t.T().Fatalf("Error creating output directory: %v", err)
119+
}
120+
t.outputDir = dir
121+
} else {
122+
t.T().Logf("Creating ephemeral output directory as %s env variable is unset", CodeFlareTestOutputDir)
123+
t.outputDir = t.T().TempDir()
124+
}
125+
t.T().Logf("Output directory has been created at: %s", t.outputDir)
126+
})
127+
return t.outputDir
128+
}
129+
97130
func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Namespace {
98131
t.T().Helper()
99132
namespace := createTestNamespace(t, options...)

‎test/support/utils.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,26 @@ limitations under the License.
1616

1717
package support
1818

19+
import (
20+
"io/fs"
21+
"os"
22+
"path"
23+
24+
"github.com/onsi/gomega"
25+
)
26+
1927
func Ptr[T any](v T) *T {
2028
return &v
2129
}
30+
31+
type OutputType string
32+
33+
const (
34+
Log OutputType = "log"
35+
)
36+
37+
func WriteToOutputDir(t Test, fileName string, fileType OutputType, data []byte) {
38+
t.T().Helper()
39+
t.Expect(os.WriteFile(path.Join(t.OutputDir(), fileName+"."+string(fileType)), data, fs.ModePerm)).
40+
To(gomega.Succeed())
41+
}

0 commit comments

Comments
 (0)