Merge pull request #7342 from keithjgrant/6618-job-list-ws

Jobs List websockets

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.gitignore

@@ -31,6 +31,7 @@ awx/ui/templates/ui/installing.html
 awx/ui_next/node_modules/
 awx/ui_next/coverage/
 awx/ui_next/build
+awx/ui_next/.env.local
 rsyslog.pid
 /tower-license
 /tower-license/**

awx/ui_next/package-lock.json

@@ -8988,6 +8988,12 @@
         }
       }
     },
+    "jest-websocket-mock": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/jest-websocket-mock/-/jest-websocket-mock-2.0.2.tgz",
+      "integrity": "sha512-SFTUI8O/LDGqROOMnfAzbrrX5gQ8GDhRqkzVrt8Y67evnFKccRPFI3ymS05tKcMONvVfxumat4pX/LRjM/CjVg==",
+      "dev": true
+    },
     "jest-worker": {
       "version": "24.9.0",
       "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-24.9.0.tgz",
@@ -9893,6 +9899,15 @@
         "minimist": "^1.2.5"
       }
     },
+    "mock-socket": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/mock-socket/-/mock-socket-9.0.3.tgz",
+      "integrity": "sha512-SxIiD2yE/By79p3cNAAXyLQWTvEFNEzcAO7PH+DzRqKSFaplAPFjiQLmw8ofmpCsZf+Rhfn2/xCJagpdGmYdTw==",
+      "dev": true,
+      "requires": {
+        "url-parse": "^1.4.4"
+      }
+    },
     "moo": {
       "version": "0.5.1",
       "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.1.tgz",

awx/ui_next/package.json

@@ -72,6 +72,8 @@
     "eslint-plugin-react": "^7.11.1",
     "eslint-plugin-react-hooks": "^2.2.0",
     "http-proxy-middleware": "^1.0.3",
+    "jest-websocket-mock": "^2.0.2",
+    "mock-socket": "^9.0.3",
     "prettier": "^1.18.2"
   },
   "jest": {

awx/ui_next/src/components/JobList/JobList.jsx

@@ -11,6 +11,7 @@ import PaginatedDataList, { ToolbarDeleteButton } from '../PaginatedDataList';
 import useRequest, { useDeleteItems } from '../../util/useRequest';
 import { getQSConfig, parseQueryString } from '../../util/qs';
 import JobListItem from './JobListItem';
+import useWsJobs from './useWsJobs';
 import {
   AdHocCommandsAPI,
   InventoryUpdatesAPI,
@@ -36,35 +37,39 @@ function JobList({ i18n, defaultParams, showTypeColumn = false }) {
 
   const [selected, setSelected] = useState([]);
   const location = useLocation();
-
   const {
-    result: { jobs, itemCount },
+    result: { results, count },
     error: contentError,
     isLoading,
     request: fetchJobs,
   } = useRequest(
-    useCallback(async () => {
-      const params = parseQueryString(QS_CONFIG, location.search);
-
-      const {
-        data: { count, results },
-      } = await UnifiedJobsAPI.read({ ...params });
-
-      return {
-        itemCount: count,
-        jobs: results,
-      };
-    }, [location]), // eslint-disable-line react-hooks/exhaustive-deps
-    {
-      jobs: [],
-      itemCount: 0,
-    }
+    useCallback(
+      async () => {
+        const params = parseQueryString(QS_CONFIG, location.search);
+        const { data } = await UnifiedJobsAPI.read({ ...params });
+        return data;
+      },
+      [location] // eslint-disable-line react-hooks/exhaustive-deps
+    ),
+    { results: [], count: 0 }
   );
-
   useEffect(() => {
     fetchJobs();
   }, [fetchJobs]);
 
+  // TODO: update QS_CONFIG to be safe for deps array
+  const fetchJobsById = useCallback(
+    async ids => {
+      const params = parseQueryString(QS_CONFIG, location.search);
+      params.id__in = ids.join(',');
+      const { data } = await UnifiedJobsAPI.read(params);
+      return data.results;
+    },
+    [location.search] // eslint-disable-line react-hooks/exhaustive-deps
+  );
+
+  const jobs = useWsJobs(results, fetchJobsById, QS_CONFIG);
+
   const isAllSelected = selected.length === jobs.length && selected.length > 0;
   const {
     isLoading: isDeleteLoading,
@@ -125,7 +130,7 @@ function JobList({ i18n, defaultParams, showTypeColumn = false }) {
           contentError={contentError}
           hasContentLoading={isLoading || isDeleteLoading}
           items={jobs}
-          itemCount={itemCount}
+          itemCount={count}
           pluralizedItemName={i18n._(t`Jobs`)}
           qsConfig={QS_CONFIG}
           onRowClick={handleSelect}

awx/ui_next/src/components/JobList/JobList.test.jsx

@@ -105,6 +105,16 @@ function waitForLoaded(wrapper) {
 }
 
 describe('<JobList />', () => {
+  let debug;
+  beforeEach(() => {
+    debug = global.console.debug; // eslint-disable-line prefer-destructuring
+    global.console.debug = () => {};
+  });
+
+  afterEach(() => {
+    global.console.debug = debug;
+  });
+
   test('initially renders succesfully', async () => {
     let wrapper;
     await act(async () => {

awx/ui_next/src/components/JobList/JobListItem.jsx

@@ -75,7 +75,7 @@ function JobListItem({
                 ]
               : []),
             <DataListCell key="finished">
-              {formatDateString(job.finished)}
+              {job.finished ? formatDateString(job.finished) : ''}
             </DataListCell>,
           ]}
         />

awx/ui_next/src/components/JobList/sortJobs.js

@@ -0,0 +1,78 @@
+const sortFns = {
+  finished: byFinished,
+  id: byId,
+  name: byName,
+  created_by__id: byCreatedBy,
+  unified_job_template__project__id: byProject,
+  started: byStarted,
+};
+
+export default function sortJobs(jobs, params) {
+  const { order_by = '-finished', page_size = 20 } = params;
+  const key = order_by.replace('-', '');
+  const fn = sortFns[key];
+  if (!fn) {
+    return jobs.slice(0, page_size);
+  }
+
+  const sorted = order_by[0] === '-' ? jobs.sort(reverse(fn)) : jobs.sort(fn);
+  return sorted.slice(0, page_size);
+}
+
+function reverse(fn) {
+  return (a, b) => fn(a, b) * -1;
+}
+
+function byFinished(a, b) {
+  if (!a.finished) {
+    return 1;
+  }
+  if (!b.finished) {
+    return -1;
+  }
+  return sort(new Date(a.finished), new Date(b.finished));
+}
+
+function byStarted(a, b) {
+  if (!a.started) {
+    return 1;
+  }
+  if (!b.started) {
+    return -1;
+  }
+  return sort(new Date(a.started), new Date(b.started));
+}
+
+function byId(a, b) {
+  return sort(a.id, b.id);
+}
+
+function byName(a, b) {
+  return sort(a.name, b.name);
+}
+
+function byCreatedBy(a, b) {
+  const nameA = a.summary_fields?.created_by?.id;
+  const nameB = b.summary_fields?.created_by?.id;
+  return sort(nameA, nameB) * -1;
+}
+
+function byProject(a, b) {
+  return sort(a.unified_job_template, b.unified_job_template);
+}
+
+function sort(a, b) {
+  if (!a) {
+    return -1;
+  }
+  if (!b) {
+    return 1;
+  }
+  if (a < b) {
+    return -1;
+  }
+  if (a > b) {
+    return 1;
+  }
+  return 0;
+}

awx/ui_next/src/components/JobList/useThrottle.js

@@ -0,0 +1,21 @@
+import { useState, useEffect, useRef } from 'react';
+
+export default function useThrottle(value, limit) {
+  const [throttledValue, setThrottledValue] = useState(value);
+  const lastRan = useRef(Date.now());
+
+  useEffect(() => {
+    const handler = setTimeout(() => {
+      if (Date.now() - lastRan.current >= limit) {
+        setThrottledValue(value);
+        lastRan.current = Date.now();
+      }
+    }, limit - (Date.now() - lastRan.current));
+
+    return () => {
+      clearTimeout(handler);
+    };
+  }, [value, limit]);
+
+  return throttledValue;
+}

awx/ui_next/src/components/JobList/useWsJobs.js

@@ -0,0 +1,119 @@
+import { useState, useEffect, useRef } from 'react';
+import { useLocation } from 'react-router-dom';
+import useThrottle from './useThrottle';
+import { parseQueryString } from '../../util/qs';
+import sortJobs from './sortJobs';
+
+export default function useWsJobs(initialJobs, fetchJobsById, qsConfig) {
+  const location = useLocation();
+  const [jobs, setJobs] = useState(initialJobs);
+  const [lastMessage, setLastMessage] = useState(null);
+  const [jobsToFetch, setJobsToFetch] = useState([]);
+  const throttledJobsToFetch = useThrottle(jobsToFetch, 5000);
+
+  useEffect(() => {
+    setJobs(initialJobs);
+  }, [initialJobs]);
+
+  const enqueueJobId = id => {
+    if (!jobsToFetch.includes(id)) {
+      setJobsToFetch(ids => ids.concat(id));
+    }
+  };
+  useEffect(() => {
+    (async () => {
+      if (!throttledJobsToFetch.length) {
+        return;
+      }
+      setJobsToFetch([]);
+      const newJobs = await fetchJobsById(throttledJobsToFetch);
+      const deduplicated = newJobs.filter(
+        job => !jobs.find(j => j.id === job.id)
+      );
+      if (deduplicated.length) {
+        const params = parseQueryString(qsConfig, location.search);
+        setJobs(sortJobs([...deduplicated, ...jobs], params));
+      }
+    })();
+  }, [throttledJobsToFetch, fetchJobsById]); // eslint-disable-line react-hooks/exhaustive-deps
+
+  const ws = useRef(null);
+
+  useEffect(() => {
+    if (!lastMessage || !lastMessage.unified_job_id) {
+      return;
+    }
+    const params = parseQueryString(qsConfig, location.search);
+    const filtersApplied = Object.keys(params).length > 4;
+    if (
+      filtersApplied &&
+      !['completed', 'failed', 'error'].includes(lastMessage.status)
+    ) {
+      return;
+    }
+
+    const jobId = lastMessage.unified_job_id;
+    const index = jobs.findIndex(j => j.id === jobId);
+    if (index > -1) {
+      setJobs(sortJobs(updateJob(jobs, index, lastMessage), params));
+    } else {
+      enqueueJobId(lastMessage.unified_job_id);
+    }
+  }, [lastMessage]); // eslint-disable-line react-hooks/exhaustive-deps
+
+  useEffect(() => {
+    ws.current = new WebSocket(`wss://${window.location.host}/websocket/`);
+
+    const connect = () => {
+      const xrftoken = `; ${document.cookie}`
+        .split('; csrftoken=')
+        .pop()
+        .split(';')
+        .shift();
+      ws.current.send(
+        JSON.stringify({
+          xrftoken,
+          groups: {
+            jobs: ['status_changed'],
+            schedules: ['changed'],
+            control: ['limit_reached_1'],
+          },
+        })
+      );
+    };
+    ws.current.onopen = connect;
+
+    ws.current.onmessage = e => {
+      setLastMessage(JSON.parse(e.data));
+    };
+
+    ws.current.onclose = e => {
+      // eslint-disable-next-line no-console
+      console.debug('Socket closed. Reconnecting...', e);
+      setTimeout(() => {
+        connect();
+      }, 1000);
+    };
+
+    ws.current.onerror = err => {
+      // eslint-disable-next-line no-console
+      console.debug('Socket error: ', err, 'Disconnecting...');
+      ws.current.close();
+    };
+
+    return () => {
+      ws.current.close();
+    };
+  }, []);
+
+  return jobs;
+}
+
+function updateJob(jobs, index, message) {
+  const job = {
+    ...jobs[index],
+    status: message.status,
+    finished: message.finished,
+  };
+  return [...jobs.slice(0, index), job, ...jobs.slice(index + 1)];
+}

awx/ui_next/src/components/JobList/useWsJobs.test.jsx

@@ -0,0 +1,127 @@
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import WS from 'jest-websocket-mock';
+import { mountWithContexts } from '../../../testUtils/enzymeHelpers';
+import useWsJobs from './useWsJobs';
+
+/*
+  Jest mock timers don’t play well with jest-websocket-mock,
+  so we'll stub out throttling to resolve immediately
+*/
+jest.mock('./useThrottle', () => ({
+  __esModule: true,
+  default: jest.fn(val => val),
+}));
+
+function TestInner() {
+  return <div />;
+}
+function Test({ jobs, fetch }) {
+  const qsConfig = {};
+  const syncedJobs = useWsJobs(jobs, fetch, qsConfig);
+  return <TestInner jobs={syncedJobs} />;
+}
+
+describe('useWsJobs hook', () => {
+  let debug;
+  let wrapper;
+  beforeEach(() => {
+    debug = global.console.debug; // eslint-disable-line prefer-destructuring
+    global.console.debug = () => {};
+  });
+
+  afterEach(() => {
+    global.console.debug = debug;
+  });
+
+  test('should return jobs list', () => {
+    const jobs = [{ id: 1 }];
+    wrapper = mountWithContexts(<Test jobs={jobs} />);
+
+    expect(wrapper.find('TestInner').prop('jobs')).toEqual(jobs);
+    WS.clean();
+  });
+
+  test('should establish websocket connection', async () => {
+    global.document.cookie = 'csrftoken=abc123';
+    const mockServer = new WS('wss://localhost/websocket/');
+
+    const jobs = [{ id: 1 }];
+    await act(async () => {
+      wrapper = await mountWithContexts(<Test jobs={jobs} />);
+    });
+
+    await mockServer.connected;
+    await expect(mockServer).toReceiveMessage(
+      JSON.stringify({
+        xrftoken: 'abc123',
+        groups: {
+          jobs: ['status_changed'],
+          schedules: ['changed'],
+          control: ['limit_reached_1'],
+        },
+      })
+    );
+    WS.clean();
+  });
+
+  test('should update job status', async () => {
+    global.document.cookie = 'csrftoken=abc123';
+    const mockServer = new WS('wss://localhost/websocket/');
+
+    const jobs = [{ id: 1, status: 'running' }];
+    await act(async () => {
+      wrapper = await mountWithContexts(<Test jobs={jobs} />);
+    });
+
+    await mockServer.connected;
+    await expect(mockServer).toReceiveMessage(
+      JSON.stringify({
+        xrftoken: 'abc123',
+        groups: {
+          jobs: ['status_changed'],
+          schedules: ['changed'],
+          control: ['limit_reached_1'],
+        },
+      })
+    );
+    expect(wrapper.find('TestInner').prop('jobs')[0].status).toEqual('running');
+    act(() => {
+      mockServer.send(
+        JSON.stringify({
+          unified_job_id: 1,
+          status: 'successful',
+        })
+      );
+    });
+    wrapper.update();
+
+    expect(wrapper.find('TestInner').prop('jobs')[0].status).toEqual(
+      'successful'
+    );
+    WS.clean();
+  });
+
+  test('should fetch new job', async () => {
+    global.document.cookie = 'csrftoken=abc123';
+    const mockServer = new WS('wss://localhost/websocket/');
+    const jobs = [{ id: 1 }];
+    const fetch = jest.fn(() => []);
+    await act(async () => {
+      wrapper = await mountWithContexts(<Test jobs={jobs} fetch={fetch} />);
+    });
+
+    await mockServer.connected;
+    act(() => {
+      mockServer.send(
+        JSON.stringify({
+          unified_job_id: 2,
+          status: 'running',
+        })
+      );
+    });
+
+    expect(fetch).toHaveBeenCalledWith([2]);
+    WS.clean();
+  });
+});