Repository: gatsbyjs/gatsby
File: packages/gatsby-source-drupal/src/gatsby-node.js (JavaScript)

Code before:
    )
    let requireFullRebuild = false

    drupalFetchIncrementalActivity.start()

    try {
      // Hit fastbuilds endpoint with the lastFetched date.
      const data = await axios.get(
        `${baseUrl}/gatsby-fastbuilds/sync/${lastFetched}`,
        {
          auth: basicAuth,
          headers,
          params,
        }
      )

      if (data.data.status === -1) {
        reporter.info(`Unable to pull incremental data changes from Drupal`)
        setPluginStatus({ lastFetched: data.data.timestamp })
        requireFullRebuild = true
      } else {
        // Touch nodes so they are not garbage collected by Gatsby.
        const existingNodes = getNodes().filter(
          n => n.internal.owner === `gatsby-source-drupal`
        )
        existingNodes.forEach(n => touchNode({ nodeId: n.id }))

        // Process sync data from Drupal.
        let nodesToSync = data.data.entities
        for (const nodeSyncData of nodesToSync) {
          if (nodeSyncData.action === `delete`) {
            actions.deleteNode({ node: getNode(createNodeId(nodeSyncData.id)) })
          } else {
            // The data could be a single Drupal entity or an array of Drupal
            // entities to update.
            let nodesToUpdate = nodeSyncData.data
            if (!Array.isArray(nodeSyncData.data)) {
              nodesToUpdate = [nodeSyncData.data]
            }

            for (const nodeToUpdate of nodesToUpdate) {
              await handleWebhookUpdate(
                {
                  nodeToUpdate,
                  actions,
                  cache,
                  createNodeId,
                  createContentDigest,
                  getCache,
                  getNode,
                  reporter,

Review comment (performance):
I'm a little weiry of loop-in-a-loop. Even if this is incremental, could this worst case still pull in a million nodes? What happens at a cold build? This might just be the way it is but can you double check to confirm that the performance scales linearly with the number of nodes?


Code after:
      // Hit fastbuilds endpoint with the lastFetched date.
      const data = await axios.get(
        `${baseUrl}/gatsby-fastbuilds/sync/${lastFetched}`,
        {
          auth: basicAuth,
          headers,
          params,
        }
      )

      if (data.data.status === -1) {
        // The incremental data is expired or this is the first fetch.
        reporter.info(`Unable to pull incremental data changes from Drupal`)
        setPluginStatus({ lastFetched: data.data.timestamp })
        requireFullRebuild = true
      } else {
        // Touch nodes so they are not garbage collected by Gatsby.
        getNodes().forEach(node => {
          if (node.internal.owner === `gatsby-source-drupal`) {
            touchNode({ nodeId: node.id })
          }
        })

        // Process sync data from Drupal.
        let nodesToSync = data.data.entities
        for (const nodeSyncData of nodesToSync) {
          if (nodeSyncData.action === `delete`) {
            actions.deleteNode({ node: getNode(createNodeId(nodeSyncData.id)) })
          } else {
            // The data could be a single Drupal entity or an array of Drupal
            // entities to update.
            let nodesToUpdate = nodeSyncData.data
            if (!Array.isArray(nodeSyncData.data)) {
              nodesToUpdate = [nodeSyncData.data]
            }

            for (const nodeToUpdate of nodesToUpdate) {
              await handleWebhookUpdate(
                {
                  nodeToUpdate,
                  actions,
                  cache,
                  createNodeId,
                  createContentDigest,
                  getCache,
                  getNode,
                  reporter,
                  store,
                },
                pluginOptions
              )