登录 免费开始

批量上传向量到 Qdrant 集合

耗时：20 分钟难度：中等

快速上传大规模数据集可能是一项挑战，但 Qdrant 提供了一些技巧来帮助您实现这一目标。

关于数据上传的第一个重要细节是，瓶颈通常位于客户端而非服务端。这意味着如果您要上传大型数据集，应优先选择高性能的客户端库。

我们建议为此目的使用我们的 Rust 客户端库，因为它是目前可用于 Qdrant 的最快客户端库。

如果您不使用 Rust，则可能需要考虑对上传过程进行并行化处理。

选择索引策略

Qdrant 会在有新数据到达时为稠密向量增量构建 HNSW 索引。这确保了快速搜索，但索引过程是内存和 CPU 密集型的。在批量摄入期间，频繁的索引更新会降低吞吐量并增加资源消耗。

为了控制这种行为并针对您的系统限制进行优化，请调整以下参数：

您的目标操作方法配置
最快上传，可容忍高内存占用完全禁用索引indexing_threshold: 0
上传期间低内存占用推迟 HNSW 图构建（推荐）m: 0
上传后更快实现索引可用保持索引启用（默认行为）m: 16, indexing_threshold: 10000 （默认）

如果索引在摄入期间被禁用，则必须在上传后重新启用索引，以激活快速 HNSW 搜索。

推迟 HNSW 图构建 (m: 0)

对于稠密向量，将 HNSW 的 m 参数设置为 0 可完全禁用索引构建。向量仍会被存储，但不会被索引，直到您稍后启用索引为止。

PUT /collections/{collection_name}
{
    "vectors": {
      "size": 768,
      "distance": "Cosine"
    },
    "hnsw_config": {
        "m": 0
    }
}

from qdrant_client import QdrantClient, models

client = QdrantClient(url="https://:6333")

client.create_collection(
    collection_name="{collection_name}",
    vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
    hnsw_config=models.HnswConfigDiff(
        m=0,
    ),
)

import { QdrantClient } from "@qdrant/js-client-rest";

const client = new QdrantClient({ host: "localhost", port: 6333 });

client.createCollection("{collection_name}", {
  vectors: {
    size: 768,
    distance: "Cosine",
  },
  hnsw_config: {
    m: 0,
  },
});

use qdrant_client::qdrant::{
    CreateCollectionBuilder, Distance, HnswConfigDiffBuilder, VectorParamsBuilder,
};
use qdrant_client::Qdrant;

let client = Qdrant::from_url("https://:6334").build()?;

client
    .create_collection(
        CreateCollectionBuilder::new("{collection_name}")
            .vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
            .hnsw_config(HnswConfigDiffBuilder::default().m(0)),
    )
    .await?;

import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.HnswConfigDiff;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.Collections.VectorsConfig;

QdrantClient client =
    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());

client
    .createCollectionAsync(
        CreateCollection.newBuilder()
            .setCollectionName("{collection_name}")
            .setVectorsConfig(
                VectorsConfig.newBuilder()
                    .setParams(
                        VectorParams.newBuilder()
                            .setSize(768)
                            .setDistance(Distance.Cosine)
                            .build())
                    .build())
            .setHnswConfig(HnswConfigDiff.newBuilder().setM(0).build())
            .build())
    .get();

using Qdrant.Client;
using Qdrant.Client.Grpc;

var client = new QdrantClient("localhost", 6334);

await client.CreateCollectionAsync(
	collectionName: "{collection_name}",
	vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
	hnswConfig: new HnswConfigDiff { M = 0 }
);

import (
	"context"

	"github.com/qdrant/go-client/qdrant"
)

client, err := qdrant.NewClient(&qdrant.Config{
	Host: "localhost",
	Port: 6334,
})

client.CreateCollection(context.Background(), &qdrant.CreateCollection{
	CollectionName: "{collection_name}",
	VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
		Size:     768,
		Distance: qdrant.Distance_Cosine,
	}),
	HnswConfig: &qdrant.HnswConfigDiff{
		M:        qdrant.PtrOf(uint64(0)),
	},
})

摄入完成后，通过将 m 设置为您生产环境的值（通常为 16 或 32）来重新启用 HNSW。

PATCH /collections/{collection_name}
{
    "vectors": {
      "size": 768,
      "distance": "Cosine"
    },
    "hnsw_config": {
        "m": 16
    }
}

from qdrant_client import QdrantClient, models

client = QdrantClient(url="https://:6333")

client.update_collection(
    collection_name="{collection_name}",
    vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
    hnsw_config=models.HnswConfigDiff(
        m=16,
    ),
)

import { QdrantClient } from "@qdrant/js-client-rest";

const client = new QdrantClient({ host: "localhost", port: 6333 });

client.updateCollection("{collection_name}", {
  vectors: {
    size: 768,
    distance: "Cosine",
  },
  hnsw_config: {
    m: 16,
  },
});

use qdrant_client::qdrant::{
    UpdateCollectionBuilder, HnswConfigDiffBuilder,
};
use qdrant_client::Qdrant;

let client = Qdrant::from_url("https://:6334").build()?;

client
    .update_collection(
        UpdateCollectionBuilder::new("{collection_name}")
            .hnsw_config(HnswConfigDiffBuilder::default().m(16)),
    )
    .await?;

import io.qdrant.client.grpc.Collections.UpdateCollection;
import io.qdrant.client.grpc.Collections.HnswConfigDiff;

QdrantClient client =
    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());

client.updateCollectionAsync(
    UpdateCollection.newBuilder()
        .setCollectionName("{collection_name}")
        .setHnswConfig(HnswConfigDiff.newBuilder().setM(16).build())
        .build())
    .get();

using Qdrant.Client;
using Qdrant.Client.Grpc;

var client = new QdrantClient("localhost", 6334);

await client.UpdateCollectionAsync(
	collectionName: "{collection_name}",
	hnswConfig: new HnswConfigDiff { M = 16 }
);

import (
	"context"

	"github.com/qdrant/go-client/qdrant"
)

qdrant.NewClient(&qdrant.Config{
	Host: "localhost",
	Port: 6334,
})

client, err := client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
	CollectionName: "{collection_name}",
	HnswConfig: &qdrant.HnswConfigDiff{
		M:        qdrant.PtrOf(uint64(16)),
	},
})

完全禁用索引 (indexing_threshold: 0)

如果您正在对大型数据集进行初始上传，则可能需要在上传期间禁用索引。这将避免对会被后续批次覆盖的向量进行不必要的索引。

将 indexing_threshold 设置为 0 可完全禁用索引。

PUT /collections/{collection_name}
{
    "vectors": {
      "size": 768,
      "distance": "Cosine"
    },
    "optimizers_config": {
        "indexing_threshold": 0
    }
}

from qdrant_client import QdrantClient, models

client = QdrantClient(url="https://:6333")

client.create_collection(
    collection_name="{collection_name}",
    vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
    optimizers_config=models.OptimizersConfigDiff(
        indexing_threshold=0,
    ),
)

import { QdrantClient } from "@qdrant/js-client-rest";

const client = new QdrantClient({ host: "localhost", port: 6333 });

client.createCollection("{collection_name}", {
  vectors: {
    size: 768,
    distance: "Cosine",
  },
  optimizers_config: {
    indexing_threshold: 0,
  },
});

use qdrant_client::qdrant::{
    OptimizersConfigDiffBuilder, UpdateCollectionBuilder,
};
use qdrant_client::Qdrant;

let client = Qdrant::from_url("https://:6334").build()?;

client
    .create_collection(
        CreateCollectionBuilder::new("{collection_name}")
            .optimizers_config(OptimizersConfigDiffBuilder::default().indexing_threshold(0)),
    )
    .await?;

import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.Collections.VectorsConfig;
import io.qdrant.client.grpc.Collections.OptimizersConfigDiff;

QdrantClient client =
    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());

client.createCollectionAsync(
    CreateCollection.newBuilder()
        .setCollectionName("{collection_name}")
        .setVectorsConfig(
            VectorsConfig.newBuilder()
                .setParams(
                    VectorParams.newBuilder()
                        .setSize(768)
                        .setDistance(Distance.Cosine)
                        .build())
                .build())
        .setOptimizersConfig(
            OptimizersConfigDiff.newBuilder()
                .setIndexingThreshold(0)
                .build())
        .build()
).get();

using Qdrant.Client;
using Qdrant.Client.Grpc;

var client = new QdrantClient("localhost", 6334);

await client.CreateCollectionAsync(
    collectionName: "{collection_name}",
    vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
    optimizersConfig: new OptimizersConfigDiff { IndexingThreshold = 0 }
);

import (
	"context"

	"github.com/qdrant/go-client/qdrant"
)

client, err := qdrant.NewClient(&qdrant.Config{
	Host: "localhost",
	Port: 6334,
})

client.CreateCollection(context.Background(), &qdrant.CreateCollection{
	CollectionName: "{collection_name}",
	VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
		Size:     768,
		Distance: qdrant.Distance_Cosine,
	}),
	OptimizersConfig: &qdrant.OptimizersConfigDiff{
		IndexingThreshold: qdrant.PtrOf(uint64(0)),
	},
})

上传完成后，您可以通过将 indexing_threshold 设置为期望值（默认值为 10000）来启用索引。

PATCH /collections/{collection_name}
{
    "optimizers_config": {
        "indexing_threshold": 10000
    }
}

from qdrant_client import QdrantClient, models

client = QdrantClient(url="https://:6333")

client.update_collection(
    collection_name="{collection_name}",
    optimizers_config=models.OptimizersConfigDiff(indexing_threshold=20000),
)

import { QdrantClient } from "@qdrant/js-client-rest";

const client = new QdrantClient({ host: "localhost", port: 6333 });

client.updateCollection("{collection_name}", {
  optimizers_config: {
    indexing_threshold: 10000,
  },
});

use qdrant_client::qdrant::{
    OptimizersConfigDiffBuilder, UpdateCollectionBuilder,
};
use qdrant_client::Qdrant;

let client = Qdrant::from_url("https://:6334").build()?;

client
    .update_collection(
        UpdateCollectionBuilder::new("{collection_name}")
            .optimizers_config(OptimizersConfigDiffBuilder::default().indexing_threshold(10000)),
    )
    .await?;

import io.qdrant.client.grpc.Collections.UpdateCollection;
import io.qdrant.client.grpc.Collections.OptimizersConfigDiff;

client.updateCollectionAsync(
    UpdateCollection.newBuilder()
        .setCollectionName("{collection_name}")
        .setOptimizersConfig(
            OptimizersConfigDiff.newBuilder()
                .setIndexingThreshold(20000)
                .build()
        )
        .build()
).get();

using Qdrant.Client;
using Qdrant.Client.Grpc;

var client = new QdrantClient("localhost", 6334);

await client.UpdateCollectionAsync(
    collectionName: "{collection_name}",
    optimizersConfig: new OptimizersConfigDiff { IndexingThreshold = 20000 }
);

import (
	"context"
	"github.com/qdrant/go-client/qdrant"
)

client, err := qdrant.NewClient(&qdrant.Config{
	Host: "localhost",
	Port: 6334,
})

client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
	CollectionName: "{collection_name}",
	OptimizersConfig: &qdrant.OptimizersConfigDiff{
		IndexingThreshold: qdrant.PtrOf(uint64(20000)),
	},
})

此时，Qdrant 将开始在后台对新分段和之前未索引的分段进行索引。

直接上传到磁盘

当您上传的向量无法全部放入内存时，您可能希望使用 memmap 支持。

在集合创建期间，可以使用 on_disk 参数为每个向量启用 memmap。这将始终把向量数据直接存储在磁盘上。这适用于摄入大量数据，对于十亿规模的基准测试至关重要。

在这种情况下不建议使用 memmap_threshold。它会要求优化器不断地将内存中的分段转换为磁盘上的 memmap 分段。这个过程较慢，而且在摄入大量数据时，优化器可能成为瓶颈。

更多相关信息请阅读配置 Memmap 存储。

并行上传到多个分片

在 Qdrant 中，每个集合都被拆分为分片。每个分片都有一个独立的预写日志 (WAL)，负责对操作进行排序。通过创建多个分片，您可以并行上传大型数据集。对于单台机器，每个分片 2 到 4 个分片是一个合理的数字。

PUT /collections/{collection_name}
{
    "vectors": {
      "size": 768,
      "distance": "Cosine"
    },
    "shard_number": 2
}

from qdrant_client import QdrantClient, models

client = QdrantClient(url="https://:6333")

client.create_collection(
    collection_name="{collection_name}",
    vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
    shard_number=2,
)

import { QdrantClient } from "@qdrant/js-client-rest";

const client = new QdrantClient({ host: "localhost", port: 6333 });

client.createCollection("{collection_name}", {
  vectors: {
    size: 768,
    distance: "Cosine",
  },
  shard_number: 2,
});

use qdrant_client::qdrant::{CreateCollectionBuilder, Distance, VectorParamsBuilder};
use qdrant_client::Qdrant;

let client = Qdrant::from_url("https://:6334").build()?;

client
    .create_collection(
        CreateCollectionBuilder::new("{collection_name}")
            .vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
            .shard_number(2),
    )
    .await?;

import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.Collections.VectorsConfig;

QdrantClient client =
    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());

client
    .createCollectionAsync(
        CreateCollection.newBuilder()
            .setCollectionName("{collection_name}")
            .setVectorsConfig(
                VectorsConfig.newBuilder()
                    .setParams(
                        VectorParams.newBuilder()
                            .setSize(768)
                            .setDistance(Distance.Cosine)
                            .build())
                    .build())
            .setShardNumber(2)
            .build())
    .get();

using Qdrant.Client;
using Qdrant.Client.Grpc;

var client = new QdrantClient("localhost", 6334);

await client.CreateCollectionAsync(
	collectionName: "{collection_name}",
	vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
	shardNumber: 2
);

import (
	"context"

	"github.com/qdrant/go-client/qdrant"
)

client, err := qdrant.NewClient(&qdrant.Config{
	Host: "localhost",
	Port: 6334,
})

client.CreateCollection(context.Background(), &qdrant.CreateCollection{
	CollectionName: "{collection_name}",
	VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
		Size:     768,
		Distance: qdrant.Distance_Cosine,
	}),
	ShardNumber: qdrant.PtrOf(uint32(2)),
})
此页面有用吗？

感谢您的反馈！🙏

听到这个消息我们很遗憾。😔 您可以在 GitHub 上编辑此页面，或创建一个 GitHub Issue。

本页内容