How can I add data in chunks to a d3 force directed graph?

I am trying to add nodes to a D3 force directed graph dynamically. I believe I am close, but am not understanding some critical concept related to how selections in D3 work.

If I call my addData function directly and add all of the nodes at once, everything works as expected.

However, if I use the addDataInChunks, the first two nodes and first link are added, but after that no further nodes appear in the graph. Additionally, the ability to drag nodes around is lost as soon as the second chunk is added to the graph.

What have I done wrong? How does my code need to change so this will work?

complete, sample .html file

<html>
  <head>
    <script src="https://d3js.org/d3.v7.min.js" charset="utf-8"></script>
  </head>

  <style>
    .graph {
      width: 750px;
      height: 400px;
    }
  </style>

  <script type="module">
    const nodeData = [
      {
        id: 'node 1',
        added: false,
      },
      {
        id: 'node 2',
        added: false,
      },
      {
        id: 'node 3',
        added: false,
      },
      {
        id: 'node 4',
        added: false,
      },
    ]

    const linkData = [
      {
        linkID: 'link 1',
        added: false,
        source: 'node 1',
        target: 'node 2',
      },
      {
        linkID: 'link 2',
        added: false,
        source: 'node 1',
        target: 'node 3',
      },
      {
        linkID: 'link 3',
        added: false,
        source: 'node 3',
        target: 'node 4',
      },
    ]

    //
    //
    //

    let svg = null
    let nodeContainer = null
    let linkContainer = null
    let simulation = null

    const width = 750
    const height = 400
    const nodeCount = 10
    const svgNS = d3.namespace('svg:text').space

    function setupGraph() {
      svg = d3.select('#chart').call(d3.zoom().on('zoom', zoomed)).append('g').attr('id', 'container')
      nodeContainer = svg.append('g').attr('stroke', '#fff').attr('stroke-width', 1.5).attr('id', 'nodes')
      linkContainer = svg.append('g').attr('stroke', '#999').attr('stroke-opacity', 0.6).attr('id', 'links')
    }

    const simulationNodes = []
    const simulationLinks = []

    function addData(addNodes, addLinks) {
      const links = addLinks.map((d) => ({ ...d }))
      const nodes = addNodes.map((d, index) => ({ ...d }))

      console.log(`๐Ÿš€ ~ nodes:`, nodes)
      console.log(`๐Ÿš€ ~ links:`, links)

      simulationNodes.push(...nodes)
      simulationLinks.push(...links)

      simulation = d3
        .forceSimulation(simulationNodes)
        .force(
          'link',
          d3
            .forceLink(simulationLinks)
            .id((d) => d.id)
            .distance((d) => 50)
        )
        .force('charge', d3.forceManyBody().strength(-400))
        .force('x', d3.forceX())
        .force('y', d3.forceY())
        .on('tick', ticked)

      nodeContainer = nodeContainer
        .selectAll('circle')
        .data(nodes)
        .join((enter) => {
          return enter.append((d) => {
            const circleElement = document.createElementNS(svgNS, 'circle')

            circleElement.setAttribute('r', 16)
            circleElement.setAttribute('fill', '#00ff00')
            circleElement.setAttribute('stroke', '#ffffff')
            circleElement.setAttribute('stroke-width', '3')

            return circleElement
          })
        })

      nodeContainer.append('title').text((d) => `hello ${d.id}`)
      nodeContainer.call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended))

      linkContainer = linkContainer.selectAll('line').data(links).join('line').attr('stroke-width', 1)
    }

    setupGraph()

    // addData(nodeData, linkData)

    async function addDataInChunks(allNodes, allLinks) {
      const timeout = 7000

      let nodeChunk = allNodes.slice(0, 2)
      let linkChunk = allLinks.slice(0, 1)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(2, 3)
      linkChunk = allLinks.slice(1, 2)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(3, 4)
      linkChunk = allLinks.slice(2, 3)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))

      console.log('addDataInChunks finished')
    }

    addDataInChunks(nodeData, linkData)

    //
    // Misc Functions
    //

    function zoomed(transform) {
      const t = transform.transform

      const container = this.getElementsByTagNameNS(svgNS, 'g')[0]
      const transformString = 'translate(' + t.x + ',' + t.y + ') scale(' + t.k + ')'

      container.setAttribute('transform', transformString)
    }

    function ticked() {
      linkContainer
        .attr('x1', (d) => d.source.x)
        .attr('y1', (d) => d.source.y)
        .attr('x2', (d) => d.target.x)
        .attr('y2', (d) => d.target.y)

      nodeContainer.each(function (d) {
        this.setAttribute('cx', d.x)
        this.setAttribute('cy', d.y)
      })
    }

    function dragstarted(event) {
      if (!event.active) simulation.alphaTarget(0.3).restart()
      event.subject.fx = event.subject.x
      event.subject.fy = event.subject.y
    }

    function dragged(event) {
      event.subject.fx = event.x
      event.subject.fy = event.y
    }

    function dragended(event) {
      if (!event.active) simulation.alphaTarget(0)
      event.subject.fx = null
      event.subject.fy = null
    }
  </script>
  <svg ref="chart" id="chart" class="graph" style="background-color: #141621"></svg>
</html>

I found and am trying to use Adding Nodes to a Force-Directed Graph / Neil Mayle | Observable as a guide. Hopefully, what is there still applies. I have gotten closer (I think) to the solution, but this still isnโ€™t working.

The HTML it is generating looks like:

I am getting the four circles in there. There is only a single line. For some strange reason, the 's are being added multiple times.

<html>
  <head>
    <script src="https://d3js.org/d3.v7.min.js" charset="utf-8"></script>
  </head>

  <style>
    .graph {
      width: 750px;
      height: 400px;
    }
  </style>

  <script type="module">
    const nodeData = [
      {
        id: 'node 1',
        added: false,
      },
      {
        id: 'node 2',
        added: false,
      },
      {
        id: 'node 3',
        added: false,
      },
      {
        id: 'node 4',
        added: false,
      },
    ]

    const linkData = [
      {
        linkID: 'link 1',
        added: false,
        source: 'node 1',
        target: 'node 2',
      },
      {
        linkID: 'link 2',
        added: false,
        source: 'node 1',
        target: 'node 3',
      },
      {
        linkID: 'link 3',
        added: false,
        source: 'node 3',
        target: 'node 4',
      },
    ]

    //
    //
    //

    let svg = null
    let nodeContainer = null
    let linkContainer = null
    let simulation = null

    const width = 750
    const height = 400
    const nodeCount = 10
    const svgNS = d3.namespace('svg:text').space

    function setupGraph() {
      svg = d3.select('#chart').call(d3.zoom().on('zoom', zoomed)).append('g').attr('id', 'container')
      svg.append('g').attr('stroke', '#fff').attr('stroke-width', 1.5).attr('id', 'nodes')
      svg.append('g').attr('stroke', '#999').attr('stroke-opacity', 0.6).attr('id', 'links')
    }

    const simulationNodes = []
    const simulationLinks = []

    function addData(addNodes, addLinks) {
      const links = addLinks.map((d) => ({ ...d }))
      const nodes = addNodes.map((d, index) => ({ ...d }))

      console.log(`๐Ÿš€ ~ nodes:`, nodes)
      console.log(`๐Ÿš€ ~ links:`, links)

      simulationNodes.push(...nodes)
      simulationLinks.push(...links)

      simulation = d3
        .forceSimulation(simulationNodes)
        .force(
          'link',
          d3
            .forceLink(simulationLinks)
            .id((d) => d.id)
            .distance((d) => 50)
        )
        .force('charge', d3.forceManyBody().strength(-400))
        .force('x', d3.forceX(width / 2))
        .force('y', d3.forceY(height / 2))
        .on('tick', ticked)

      //
      //
      //

      nodeContainer = svg.select('#nodes').selectAll('circle').data(simulationNodes)

      const newNodes = nodeContainer
        .join('circle')
        .attr('fill', '#00ff00')
        .attr('r', 16)
        .call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended))

      newNodes.append('title').text((d) => `hello ${d.id}`)

      nodeContainer = nodeContainer.merge(newNodes)
      nodeContainer = svg.select('#nodes').selectAll('circle')

      // nodeContainer.append('title').text((d) => `hello ${d.id}`)
      // nodeContainer.call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended))

      //
      //
      //
      linkContainer = svg.select('#links').selectAll('line').data(links)
      const newLinks = linkContainer.join('line').attr('stroke-width', 1)

      linkContainer = linkContainer.merge(newLinks)
      linkContainer = svg.select('#links').selectAll('line')
    }

    setupGraph()

    // addData(nodeData, linkData)

    async function addDataInChunks(allNodes, allLinks) {
      const timeout = 7000

      let nodeChunk = allNodes.slice(0, 2)
      let linkChunk = allLinks.slice(0, 1)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(2, 3)
      linkChunk = allLinks.slice(1, 2)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(3, 4)
      linkChunk = allLinks.slice(2, 3)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))

      console.log('addDataInChunks finished')
    }

    addDataInChunks(nodeData, linkData)

    //
    // Misc Functions
    //

    function zoomed(transform) {
      const t = transform.transform

      const container = this.getElementsByTagNameNS(svgNS, 'g')[0]
      const transformString = 'translate(' + t.x + ',' + t.y + ') scale(' + t.k + ')'

      container.setAttribute('transform', transformString)
    }

    function ticked() {
      linkContainer
        .attr('x1', (d) => d.source.x)
        .attr('y1', (d) => d.source.y)
        .attr('x2', (d) => d.target.x)
        .attr('y2', (d) => d.target.y)

      nodeContainer.each(function (d) {
        this.setAttribute('cx', d.x)
        this.setAttribute('cy', d.y)
      })
    }

    function dragstarted(event) {
      if (!event.active) simulation.alphaTarget(0.3).restart()
      event.subject.fx = event.subject.x
      event.subject.fy = event.subject.y
    }

    function dragged(event) {
      event.subject.fx = event.x
      event.subject.fy = event.y
    }

    function dragended(event) {
      if (!event.active) simulation.alphaTarget(0)
      event.subject.fx = null
      event.subject.fy = null
    }
  </script>
  <svg ref="chart" id="chart" class="graph" style="background-color: #141621"></svg>
</html>

Some more progress. This one is seems to be getting closer. However, I do not believe it is working as intended. I think the entire graph is being regenerated. It should be the case that a new node is appended and the graph adjusted to fit the new node in. Or am on wrong?

I am still finding it strange that some of the nodes have multiple elements.

Here is the html being generated:

Updated example:

<html>
  <head>
    <script src="https://d3js.org/d3.v7.min.js" charset="utf-8"></script>
  </head>

  <style>
    .graph {
      width: 750px;
      height: 400px;
    }
  </style>

  <script type="module">
    const nodeData = [
      {
        id: 'node 1',
        added: false,
      },
      {
        id: 'node 2',
        added: false,
      },
      {
        id: 'node 3',
        added: false,
      },
      {
        id: 'node 4',
        added: false,
      },
    ]

    const linkData = [
      {
        linkID: 'link 1',
        added: false,
        source: 'node 1',
        target: 'node 2',
      },
      {
        linkID: 'link 2',
        added: false,
        source: 'node 1',
        target: 'node 3',
      },
      {
        linkID: 'link 3',
        added: false,
        source: 'node 3',
        target: 'node 4',
      },
    ]

    //
    //
    //

    let svg = null
    let nodeContainer = null
    let linkContainer = null
    let simulation = null

    const width = 750
    const height = 400
    const nodeCount = 10
    const svgNS = d3.namespace('svg:text').space

    function setupGraph() {
      svg = d3.select('#chart').call(d3.zoom().on('zoom', zoomed)).append('g').attr('id', 'container')
      svg.append('g').attr('stroke', '#fff').attr('stroke-width', 1.5).attr('id', 'nodes')
      svg.append('g').attr('stroke', '#999').attr('stroke-opacity', 0.6).attr('id', 'links')
    }

    const simulationNodes = []
    const simulationLinks = []

    function addData(addNodes, addLinks) {
      const links = addLinks.map((d) => ({ ...d }))
      const nodes = addNodes.map((d, index) => ({ ...d }))

      console.log(`๐Ÿš€ ~ nodes:`, nodes)
      console.log(`๐Ÿš€ ~ links:`, links)

      simulationNodes.push(...nodes)
      simulationLinks.push(...links)

      simulation = d3
        .forceSimulation(simulationNodes)
        .force(
          'link',
          d3
            .forceLink(simulationLinks)
            .id((d) => d.id)
            .distance((d) => 50)
        )
        .force('charge', d3.forceManyBody().strength(-400))
        .force('x', d3.forceX(width / 2))
        .force('y', d3.forceY(height / 2))
        .on('tick', ticked)

      //
      //
      //

      nodeContainer = svg.select('#nodes').selectAll('circle').data(simulationNodes)

      const newNodes = nodeContainer
        .join('circle')
        .attr('fill', '#00ff00')
        .attr('r', 16)
        .call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended))
        .append('title')
        .text((d) => `hello ${d.id}`)

      // nodeContainer = svg.select('#nodes').selectAll('circle')

      // nodeContainer = nodeContainer.merge(newNodes)

      nodeContainer = svg.select('#nodes').selectAll('circle')

      // nodeContainer.append('title').text((d) => `hello ${d.id}`)
      // nodeContainer.call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended))

      //
      //
      //
      linkContainer = svg.select('#links').selectAll('line').data(simulationLinks)
      const newLinks = linkContainer.join('line').attr('stroke-width', 1)

      // linkContainer = linkContainer.merge(newLinks)
      linkContainer = svg.select('#links').selectAll('line')
    }

    setupGraph()

    // addData(nodeData, linkData)

    async function addDataInChunks(allNodes, allLinks) {
      const timeout = 7000

      let nodeChunk = allNodes.slice(0, 2)
      let linkChunk = allLinks.slice(0, 1)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(2, 3)
      linkChunk = allLinks.slice(1, 2)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(3, 4)
      linkChunk = allLinks.slice(2, 3)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))

      console.log('addDataInChunks finished')
    }

    addDataInChunks(nodeData, linkData)

    //
    // Misc Functions
    //

    function zoomed(transform) {
      const t = transform.transform

      const container = this.getElementsByTagNameNS(svgNS, 'g')[0]
      const transformString = 'translate(' + t.x + ',' + t.y + ') scale(' + t.k + ')'

      container.setAttribute('transform', transformString)
    }

    function ticked() {
      linkContainer
        .attr('x1', (d) => d.source.x)
        .attr('y1', (d) => d.source.y)
        .attr('x2', (d) => d.target.x)
        .attr('y2', (d) => d.target.y)

      nodeContainer.each(function (d) {
        this.setAttribute('cx', d.x)
        this.setAttribute('cy', d.y)
      })
    }

    function dragstarted(event) {
      if (!event.active) simulation.alphaTarget(0.3).restart()
      event.subject.fx = event.subject.x
      event.subject.fy = event.subject.y
    }

    function dragged(event) {
      event.subject.fx = event.x
      event.subject.fy = event.y
    }

    function dragended(event) {
      if (!event.active) simulation.alphaTarget(0)
      event.subject.fx = null
      event.subject.fy = null
    }
  </script>
  <svg ref="chart" id="chart" class="graph" style="background-color: #141621"></svg>
</html>

Here is my closest attempt yet. It is almost identical to the previous one. The primary change is that I am creating the once simulation in setupGraph rather than on each call to addData. addData will update the simulation with the new nodes, etc. I renamed some variables to hopefully add clarity. Additionally, each node gets a different color to make them easier to identify.

I do have a few questions / issues to resolve.

What I believe is happening is that I am creating all of the nodes over and over again. The evidence I have for this is that a node can end up with multiple elements. The code is written as:

      graphNodes.append('title').text((d) => `hello ${d.id}`)

so, that makes sense. However, I am not sure how to resolve this issue. It seems like I should be able to create a selection with just the new nodes to add and then merge them together. However, various attempts to use .merge have failed.

Being able to merge the new nodes should allow me to start them in the middle and then have the forces take over to figure out how the graph should be laid out. It is jarring for the nodes to start in the top left and then be pulled towards the center.

Does anyone have any advice?

<html>
  <head>
    <script src="https://d3js.org/d3.v7.min.js" charset="utf-8"></script>
  </head>

  <style>
    .graph {
      width: 750px;
      height: 400px;
    }
  </style>

  <script type="module">
    const nodeData = [
      {
        id: 'node 1',
        added: false,
        color: 'red',
      },
      {
        id: 'node 2',
        added: false,
        color: 'green',
      },
      {
        id: 'node 3',
        added: false,
        color: 'blue',
      },
      {
        id: 'node 4',
        added: false,
        color: 'black',
      },
    ]

    const linkData = [
      {
        linkID: 'link 1',
        added: false,
        source: 'node 1',
        target: 'node 2',
      },
      {
        linkID: 'link 2',
        added: false,
        source: 'node 1',
        target: 'node 3',
      },
      {
        linkID: 'link 3',
        added: false,
        source: 'node 3',
        target: 'node 4',
      },
    ]

    //
    //
    //

    let svg = null
    let graphNodes = null
    let graphLinks = null
    let simulation = null

    const width = 750
    const height = 400
    const nodeCount = 10
    const svgNS = d3.namespace('svg:text').space

    function setupGraph() {
      svg = d3.select('#chart').call(d3.zoom().on('zoom', zoomed)).append('g').attr('id', 'container')
      svg.append('g').attr('stroke', '#fff').attr('stroke-width', 1.5).attr('id', 'nodes')
      svg.append('g').attr('stroke', '#999').attr('stroke-opacity', 0.6).attr('id', 'links')

      simulation = d3
        .forceSimulation(simulationNodes)
        .force(
          'link',
          d3
            .forceLink(simulationLinks)
            .id((d) => d.id)
            .distance((d) => 50)
        )
        .force('charge', d3.forceManyBody().strength(-400))
        .force('x', d3.forceX(width / 2))
        .force('y', d3.forceY(height / 2))
    }

    const simulationNodes = []
    const simulationLinks = []

    function addData(addNodes, addLinks) {
      const links = addLinks.map((d) => ({ ...d }))
      const nodes = addNodes.map((d, index) => ({ ...d }))

      console.log(`๐Ÿš€ ~ nodes:`, nodes)
      console.log(`๐Ÿš€ ~ links:`, links)

      simulationNodes.push(...nodes)
      simulationLinks.push(...links)

      //
      //
      //

      graphNodes = svg
        .select('#nodes')
        .selectAll('circle')
        .data(simulationNodes)
        .join('circle')
        .attr('fill', (d) => d.color)
        .attr('r', 16)
        .call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended))

      graphNodes.append('title').text((d) => `hello ${d.id}`)

      //
      //
      //

      graphLinks = svg.select('#links').selectAll('line').data(simulationLinks).join('line').attr('stroke-width', 1)

      simulation.nodes(simulationNodes).on('tick', ticked)
      simulation.force('link').links(simulationLinks)
      simulation.alpha(1).restart()
    }

    setupGraph()

    // addData(nodeData, linkData)

    async function addDataInChunks(allNodes, allLinks) {
      const timeout = 7000

      let nodeChunk = allNodes.slice(0, 2)
      let linkChunk = allLinks.slice(0, 1)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(2, 3)
      linkChunk = allLinks.slice(1, 2)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))
      //
      nodeChunk = allNodes.slice(3, 4)
      linkChunk = allLinks.slice(2, 3)

      addData(nodeChunk, linkChunk)
      await new Promise((r) => setTimeout(r, timeout))

      console.log('addDataInChunks finished')
    }

    addDataInChunks(nodeData, linkData)

    //
    // Misc Functions
    //

    function zoomed(transform) {
      const t = transform.transform

      const container = this.getElementsByTagNameNS(svgNS, 'g')[0]
      const transformString = 'translate(' + t.x + ',' + t.y + ') scale(' + t.k + ')'

      container.setAttribute('transform', transformString)
    }

    function ticked() {
      graphLinks
        .attr('x1', (d) => d.source.x)
        .attr('y1', (d) => d.source.y)
        .attr('x2', (d) => d.target.x)
        .attr('y2', (d) => d.target.y)

      graphNodes.each(function (d) {
        this.setAttribute('cx', d.x)
        this.setAttribute('cy', d.y)
      })
    }

    function dragstarted(event) {
      if (!event.active) simulation.alphaTarget(0.3).restart()
      event.subject.fx = event.subject.x
      event.subject.fy = event.subject.y
    }

    function dragged(event) {
      event.subject.fx = event.x
      event.subject.fy = event.y
    }

    function dragended(event) {
      if (!event.active) simulation.alphaTarget(0)
      event.subject.fx = null
      event.subject.fy = null
    }
  </script>
  <svg ref="chart" id="chart" class="graph" style="background-color: #141621"></svg>
</html>

In case anyone is interested in the answer, I did pose the question on Stack Overflow and received the answer.

If you like, I would advocate for heading over there and upvoting the answer. I like supporting those who are helpful.

If anyone happens to have a comment on how to get the new incoming nodes to start in the center, please let me know. If I understand what is happening correctly, (enter) => โ€ฆ is going to only be called with the incoming (i.e. new) nodes. I should be able do something like

              let e = enter
                .append('circle')
                .attr('r', 16)
                .attr('fill', '#318631')
                .attr('stroke', '#7CC07C')
                .attr('stroke-width', '3');
                .attr('cx', width / 2 );
                .attr('cy', height / 2 );

basically, add cx and cy attributes to provide an initial positionโ€ฆ

I have a followup question in a new, related threadโ€ฆ