diff --git a/.github/image/Dockerfile b/.github/image/Dockerfile
index 0580d19f0..e644731e9 100644
--- a/.github/image/Dockerfile
+++ b/.github/image/Dockerfile
@@ -18,7 +18,7 @@ RUN git submodule update --init
RUN go mod download
# Download Go dependencies for Curio
-WORKDIR /app/boost
+WORKDIR /app/curio
RUN git submodule update --init
RUN go mod download
diff --git a/.gitignore b/.gitignore
index ee9bf7cf3..9f2c5e3fc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
/booster-bitswap
/docgen-md
/docgen-openrpc
+/migrate-curio
extern/filecoin-ffi/rust/target
extern/boostd-data/boostd-data
**/*.a
diff --git a/Makefile b/Makefile
index 70d74634a..2ddf9453a 100644
--- a/Makefile
+++ b/Makefile
@@ -133,6 +133,11 @@ boostci: $(BUILD_DEPS)
$(GOCC) build $(GOFLAGS) -o boostci ./cmd/boostci
.PHONY: boostci
+migrate-curio: $(BUILD_DEPS)
+ rm -f migrate-curio
+ $(GOCC) build $(GOFLAGS) -o migrate-curio ./cmd/migrate-curio
+.PHONY: boostci
+
react: validate-node-version
npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react
npm run --prefix react build
@@ -143,7 +148,7 @@ update-react: validate-node-version
npm run --prefix react build
.PHONY: react
-build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid
+build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid migrate-curio
.PHONY: build-go
build: react build-go
@@ -162,6 +167,7 @@ install-boost:
install -C ./booster-http /usr/local/bin/booster-http
install -C ./booster-bitswap /usr/local/bin/booster-bitswap
install -C ./migrate-lid /usr/local/bin/migrate-lid
+ install -C ./migrate-curio /usr/local/bin/migrate-curio
install-devnet:
install -C ./devnet /usr/local/bin/devnet
diff --git a/api/api.go b/api/api.go
index 91f4fd425..6f7a0b943 100644
--- a/api/api.go
+++ b/api/api.go
@@ -26,6 +26,7 @@ type Boost interface {
Net
// MethodGroup: Boost
+ BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) //perm:admin
BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error) //perm:admin
BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin
diff --git a/api/proxy_gen.go b/api/proxy_gen.go
index 8f3ce38a0..b98bf78bb 100644
--- a/api/proxy_gen.go
+++ b/api/proxy_gen.go
@@ -60,6 +60,8 @@ type BoostStruct struct {
BoostIndexerListMultihashes func(p0 context.Context, p1 []byte) ([]multihash.Multihash, error) `perm:"admin"`
+ BoostIndexerRemoveAll func(p0 context.Context) ([]cid.Cid, error) `perm:"admin"`
+
BoostLegacyDealByProposalCid func(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) `perm:"admin"`
BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string, p3 bool) (*ProviderDealRejectionInfo, error) `perm:"admin"`
@@ -355,6 +357,17 @@ func (s *BoostStub) BoostIndexerListMultihashes(p0 context.Context, p1 []byte) (
return *new([]multihash.Multihash), ErrNotSupported
}
+func (s *BoostStruct) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) {
+ if s.Internal.BoostIndexerRemoveAll == nil {
+ return *new([]cid.Cid), ErrNotSupported
+ }
+ return s.Internal.BoostIndexerRemoveAll(p0)
+}
+
+func (s *BoostStub) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) {
+ return *new([]cid.Cid), ErrNotSupported
+}
+
func (s *BoostStruct) BoostLegacyDealByProposalCid(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) {
if s.Internal.BoostLegacyDealByProposalCid == nil {
return *new(legacytypes.MinerDeal), ErrNotSupported
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
index e61377cd1..9b69ce54e 100644
Binary files a/build/openrpc/boost.json.gz and b/build/openrpc/boost.json.gz differ
diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go
index 0e4b0fd69..35dc919dd 100644
--- a/cmd/boostd/index.go
+++ b/cmd/boostd/index.go
@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
indexProvAnnounceLatestHttp,
indexProvAnnounceDealRemovalAd,
indexProvAnnounceDeal,
+ indexProvRemoveAllCmd,
},
}
@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
return nil
},
}
+
+var indexProvRemoveAllCmd = &cli.Command{
+ Name: "remove-all",
+ Usage: "Announce all removal ad for all contextIDs",
+ Flags: []cli.Flag{},
+ Action: func(cctx *cli.Context) error {
+ ctx := lcli.ReqContext(cctx)
+
+ // get boost api
+ napi, closer, err := bcli.GetBoostAPI(cctx)
+ if err != nil {
+ return err
+ }
+ defer closer()
+
+ // announce markets and boost deals
+ cids, err := napi.BoostIndexerRemoveAll(ctx)
+ if err != nil {
+ return err
+ }
+ for _, c := range cids {
+ fmt.Println("Published the removal ad with CID", c.String())
+ }
+ return nil
+ },
+}
diff --git a/cmd/migrate-curio/leveldb.go b/cmd/migrate-curio/leveldb.go
new file mode 100644
index 000000000..d4b7f8ebd
--- /dev/null
+++ b/cmd/migrate-curio/leveldb.go
@@ -0,0 +1,19 @@
+package main
+
+import (
+ "fmt"
+
+ "github.com/urfave/cli/v2"
+)
+
+var cleanupLevelDBCmd = &cli.Command{
+ Name: "leveldb",
+ Description: "Removes the indexes and other metadata leveldb based LID store",
+ Usage: "migrate-curio cleanup leveldb",
+ Before: before,
+ Action: func(cctx *cli.Context) error {
+ fmt.Println("Please remove the directory called 'LID' in the boost repo path to remove leveldb based LID")
+ fmt.Println("This directory can also be present outside of Boost repo if 'boostd-data' was running with a custom repo path")
+ return nil
+ },
+}
diff --git a/cmd/migrate-curio/main.go b/cmd/migrate-curio/main.go
new file mode 100644
index 000000000..ad49a4d5a
--- /dev/null
+++ b/cmd/migrate-curio/main.go
@@ -0,0 +1,88 @@
+package main
+
+import (
+ "os"
+
+ "github.com/filecoin-project/boost/build"
+ logging "github.com/ipfs/go-log/v2"
+ "github.com/urfave/cli/v2"
+)
+
+var log = logging.Logger("migrate-curio")
+
+const (
+ FlagBoostRepo = "boost-repo"
+)
+
+var FlagRepo = &cli.StringFlag{
+ Name: FlagBoostRepo,
+ EnvVars: []string{"BOOST_PATH"},
+ Usage: "boost repo path",
+ Value: "~/.boost",
+}
+
+var IsVeryVerbose bool
+
+var FlagVeryVerbose = &cli.BoolFlag{
+ Name: "vv",
+ Usage: "enables very verbose mode, useful for debugging the CLI",
+ Destination: &IsVeryVerbose,
+}
+
+func main() {
+ app := &cli.App{
+ Name: "migrate-curio",
+ Usage: "Migrate boost to Curio",
+ EnableBashCompletion: true,
+ Version: build.UserVersion(),
+ Flags: []cli.Flag{
+ FlagRepo,
+ FlagVeryVerbose,
+ &cli.StringFlag{
+ Name: "db-host",
+ EnvVars: []string{"CURIO_DB_HOST", "CURIO_HARMONYDB_HOSTS"},
+ Usage: "Command separated list of hostnames for yugabyte cluster",
+ Value: "127.0.0.1",
+ },
+ &cli.StringFlag{
+ Name: "db-name",
+ EnvVars: []string{"CURIO_DB_NAME", "CURIO_HARMONYDB_NAME"},
+ Value: "yugabyte",
+ },
+ &cli.StringFlag{
+ Name: "db-user",
+ EnvVars: []string{"CURIO_DB_USER", "CURIO_HARMONYDB_USERNAME"},
+ Value: "yugabyte",
+ },
+ &cli.StringFlag{
+ Name: "db-password",
+ EnvVars: []string{"CURIO_DB_PASSWORD", "CURIO_HARMONYDB_PASSWORD"},
+ Value: "yugabyte",
+ },
+ &cli.StringFlag{
+ Name: "db-port",
+ EnvVars: []string{"CURIO_DB_PORT", "CURIO_HARMONYDB_PORT"},
+ Value: "5433",
+ },
+ },
+ Commands: []*cli.Command{
+ migrateCmd,
+ cleanupLIDCmd,
+ },
+ }
+ app.Setup()
+
+ if err := app.Run(os.Args); err != nil {
+ _, _ = os.Stderr.WriteString("Error: " + err.Error() + "\n")
+ }
+}
+
+func before(cctx *cli.Context) error {
+ _ = logging.SetLogLevel("migrate-curio", "INFO")
+
+ if IsVeryVerbose {
+ _ = logging.SetLogLevel("migrate-curio", "DEBUG")
+ }
+
+ return nil
+}
diff --git a/cmd/migrate-curio/migrate.go b/cmd/migrate-curio/migrate.go
new file mode 100644
index 000000000..93fe4eb88
--- /dev/null
+++ b/cmd/migrate-curio/migrate.go
@@ -0,0 +1,632 @@
+package main
+
+import (
+ "bytes"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "path"
+ "time"
+
+ "github.com/filecoin-project/boost/db"
+ "github.com/filecoin-project/boost/lib/legacy"
+ "github.com/filecoin-project/boost/node/repo"
+ "github.com/filecoin-project/boost/storagemarket/types"
+ "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints"
+ "github.com/filecoin-project/boost/storagemarket/types/legacytypes"
+ transportTypes "github.com/filecoin-project/boost/transport/types"
+ "github.com/filecoin-project/curio/deps"
+ "github.com/filecoin-project/curio/harmony/harmonydb"
+ "github.com/filecoin-project/go-address"
+ "github.com/filecoin-project/go-bitfield"
+ vfsm "github.com/filecoin-project/go-ds-versioning/pkg/fsm"
+ "github.com/filecoin-project/go-state-types/abi"
+ verifreg9types "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
+ "github.com/filecoin-project/go-statemachine/fsm"
+ "github.com/filecoin-project/lotus/api/v1api"
+ "github.com/filecoin-project/lotus/blockstore"
+ "github.com/filecoin-project/lotus/chain/actors/adt"
+ "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+ ltypes "github.com/filecoin-project/lotus/chain/types"
+ lcli "github.com/filecoin-project/lotus/cli"
+ lotus_repo "github.com/filecoin-project/lotus/node/repo"
+ "github.com/ipfs/go-datastore"
+ "github.com/ipfs/go-datastore/namespace"
+ cbor "github.com/ipfs/go-ipld-cbor"
+ "github.com/mitchellh/go-homedir"
+ "github.com/urfave/cli/v2"
+ "golang.org/x/net/context"
+ "golang.org/x/xerrors"
+)
+
+var migrateCmd = &cli.Command{
+ Name: "migrate",
+ Description: "Migrate boost metadata to Curio",
+ Usage: "migrate-curio migrate",
+ Before: before,
+ Action: func(cctx *cli.Context) error {
+ repoDir, err := homedir.Expand(cctx.String(FlagBoostRepo))
+ if err != nil {
+ return err
+ }
+
+ return migrate(cctx, repoDir)
+ },
+}
+
+func migrate(cctx *cli.Context, repoDir string) error {
+ ctx := cctx.Context
+
+ r, err := lotus_repo.NewFS(repoDir)
+ if err != nil {
+ return err
+ }
+ ok, err := r.Exists()
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return fmt.Errorf("repo at '%s' is not initialized", cctx.String(FlagBoostRepo))
+ }
+
+ lr, err := r.Lock(repo.Boost)
+ if err != nil {
+ return err
+ }
+
+ mds, err := lr.Datastore(ctx, "/metadata")
+ if err != nil {
+ return err
+ }
+
+ maddrb, err := mds.Get(context.TODO(), datastore.NewKey("miner-address"))
+ if err != nil {
+ return err
+ }
+
+ maddr, err := address.NewFromBytes(maddrb)
+ if err != nil {
+ return err
+ }
+
+ // Connect to full Node
+ full, closer, err := lcli.GetFullNodeAPIV1(cctx)
+ if err != nil {
+ return xerrors.Errorf("failed to connect to full node API: %w", err)
+ }
+ defer closer()
+
+ // Connect to Harmony DB
+ hdb, err := deps.MakeDB(cctx)
+ if err != nil {
+ return xerrors.Errorf("failed to connect to harmony DB: %w", err)
+ }
+
+ dbPath := path.Join(repoDir, "boost.db?cache=shared")
+ sqldb, err := db.SqlDB(dbPath)
+ if err != nil {
+ return fmt.Errorf("opening boost sqlite db: %w", err)
+ }
+
+ mdbPath := path.Join(repoDir, "migrate-curio.db?cache=shared")
+ mdb, err := db.SqlDB(mdbPath)
+ if err != nil {
+ return fmt.Errorf("opening migrate sqlite db: %w", err)
+ }
+
+ _, err = mdb.Exec(`CREATE TABLE IF NOT EXISTS Deals (
+ ID TEXT UNIQUE,
+ DB BOOL NOT NULL DEFAULT FALSE,
+ LID BOOL NOT NULL DEFAULT FALSE,
+ Pipeline BOOL NOT NULL DEFAULT FALSE
+ );`)
+ if err != nil {
+ return fmt.Errorf("failed to create migration table: %w", err)
+ }
+
+ mActor, err := full.StateGetActor(ctx, maddr, ltypes.EmptyTSK)
+ if err != nil {
+ return fmt.Errorf("getting actor for the miner %s: %w", maddr, err)
+ }
+ astore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(full)))
+ mas, err := miner.Load(astore, mActor)
+ if err != nil {
+ return fmt.Errorf("loading miner actor state %s: %w", maddr, err)
+ }
+ liveSectors, err := miner.AllPartSectors(mas, miner.Partition.LiveSectors)
+ if err != nil {
+ return fmt.Errorf("getting live sector sets for miner %s: %w", maddr, err)
+ }
+ unProvenSectors, err := miner.AllPartSectors(mas, miner.Partition.UnprovenSectors)
+ if err != nil {
+ return fmt.Errorf("getting unproven sector sets for miner %s: %w", maddr, err)
+ }
+ activeSectors, err := bitfield.MergeBitFields(liveSectors, unProvenSectors)
+ if err != nil {
+ return fmt.Errorf("merging bitfields to generate all sealed sectors on miner %s: %w", maddr, err)
+ }
+
+ // Migrate Boost deals
+ if err := migrateBoostDeals(ctx, activeSectors, maddr, hdb, sqldb, mdb); err != nil {
+ return xerrors.Errorf("failed to migrate boost deals: %w", err)
+ }
+
+ // Migrate Legacy deal
+ if err := migrateLegacyDeals(ctx, full, activeSectors, maddr, hdb, mds, mdb); err != nil {
+ return xerrors.Errorf("failed to migrate legacy deals: %w", err)
+ }
+
+ // Migrate Direct deals
+ if err := migrateDDODeals(ctx, full, activeSectors, maddr, hdb, sqldb, mdb); err != nil {
+ return xerrors.Errorf("failed to migrate DDO deals: %w", err)
+ }
+
+ return nil
+}
+
+func migrateBoostDeals(ctx context.Context, activeSectors bitfield.BitField, maddr address.Address, hdb *harmonydb.DB, sqldb, mdb *sql.DB) error {
+ sdb := db.NewDealsDB(sqldb)
+
+ mid, err := address.IDFromAddress(maddr)
+ if err != nil {
+ return fmt.Errorf("address.IDFromAddress: %s", err)
+ }
+
+ aDeals, err := sdb.ListActive(ctx)
+ if err != nil {
+ return err
+ }
+
+ cDeals, err := sdb.ListCompleted(ctx)
+ if err != nil {
+ return err
+ }
+
+ deals := append(aDeals, cDeals...)
+
+ for i, deal := range deals {
+ if i > 0 && i%100 == 0 {
+ fmt.Printf("Migrating Boost Deals: %d / %d (%0.2f%%)\n", i, len(deals), float64(i)/float64(len(deals))*100)
+ }
+
+ llog := log.With("Boost Deal", deal.DealUuid.String())
+ // Skip deals which are before add piece
+ if deal.Checkpoint < dealcheckpoints.AddedPiece {
+ llog.Infow("Skipping as checkpoint is below add piece")
+ continue
+ }
+
+ // Skip deals which do not have chain deal ID
+ if deal.ChainDealID == 0 {
+ llog.Infow("Skipping as chain deal ID is 0")
+ continue
+ }
+
+ // Skip deals which do not have retryable error
+ if deal.Retry == types.DealRetryFatal {
+ llog.Infow("Skipping as deal retry is fatal")
+ continue
+ }
+
+ // SKip sector 0. This might cause some deals to not migrate but
+ // that is better than migrating faulty deals
+ if deal.SectorID == 0 {
+ llog.Infow("Skipping as sector ID is 0")
+ continue
+ }
+
+ // Skip if the sector for the deal is not alive
+ ok, err := activeSectors.IsSet(uint64(deal.SectorID))
+ if err != nil {
+ return err
+ }
+ if !ok {
+ llog.Infof("Skipping as sector %d is not alive anymore", deal.SectorID)
+ continue
+ }
+
+ // Skip if already migrated
+ var a, b, c bool
+ err = mdb.QueryRow(`SELECT DB, LID, Pipeline FROM Deals WHERE ID = ?`, deal.DealUuid.String()).Scan(&a, &b, &c)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("deal: %s: failed to check migration status: %w", deal.DealUuid.String(), err)
+ }
+ }
+ if a && b && c {
+ llog.Infow("Skipped as this deal is already migrated")
+ continue
+ }
+
+ propJson, err := json.Marshal(deal.ClientDealProposal.Proposal)
+ if err != nil {
+ return fmt.Errorf("deal: %s: json.Marshal(piece.DealProposal): %s", deal.DealUuid.String(), err)
+ }
+
+ sigByte, err := deal.ClientDealProposal.ClientSignature.MarshalBinary()
+ if err != nil {
+ return fmt.Errorf("deal: %s: marshal client signature: %s", deal.DealUuid.String(), err)
+ }
+
+ prop := deal.ClientDealProposal.Proposal
+
+ sProp, err := deal.SignedProposalCid()
+ if err != nil {
+ return err
+ }
+
+ tInfo := &transportTypes.HttpRequest{}
+ var headers []byte
+ if !deal.IsOffline {
+ // de-serialize transport opaque token
+ if err := json.Unmarshal(deal.Transfer.Params, tInfo); err != nil {
+ return fmt.Errorf("deal: %s: failed to de-serialize transport params bytes '%s': %s", deal.DealUuid.String(), string(deal.Transfer.Params), err)
+ }
+
+ goheaders := http.Header{}
+ for k, v := range tInfo.Headers {
+ goheaders.Set(k, v)
+ }
+
+ headers, err = json.Marshal(goheaders)
+ if err != nil {
+ return fmt.Errorf("deal: %s: failed to marshal headers: %s", deal.DealUuid.String(), err)
+ }
+ } else {
+ headers = []byte("{}")
+ }
+
+ // Cbor marshal the Deal Label manually as non-string label will result in "" with JSON marshal
+ label := prop.Label
+ propCid, err := prop.Cid()
+ if err != nil {
+ return fmt.Errorf("deal: %s: failed to calculate prop.Cid: %s", deal.DealUuid.String(), err)
+ }
+ buf := new(bytes.Buffer)
+ err = label.MarshalCBOR(buf)
+ if err != nil {
+ return fmt.Errorf("cbor marshal label: %w", err)
+
+ }
+
+ _, err = hdb.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) {
+ // Add deal to HarmonyDB
+ if !a {
+ _, err = tx.Exec(`INSERT INTO market_mk12_deals (uuid, sp_id, signed_proposal_cid,
+ proposal_signature, proposal, proposal_cid, piece_cid,
+ piece_size, offline, verified, start_epoch, end_epoch,
+ client_peer_id, fast_retrieval, announce_to_ipni, url, url_headers, chain_deal_id, publish_cid, created_at, label)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)
+ ON CONFLICT (uuid) DO NOTHING`,
+ deal.DealUuid.String(), mid, sProp.String(), sigByte, propJson, propCid, prop.PieceCID.String(),
+ prop.PieceSize, deal.IsOffline, prop.VerifiedDeal, prop.StartEpoch, prop.EndEpoch, deal.ClientPeerID.String(),
+ deal.FastRetrieval, deal.AnnounceToIPNI, tInfo.URL, headers, int64(deal.ChainDealID), deal.PublishCID.String(), deal.CreatedAt, buf.Bytes())
+
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add the deal to harmonyDB: %w", deal.DealUuid.String(), err)
+ }
+
+ // Mark deal added to harmonyDB
+ _, err = mdb.Exec(`INSERT INTO Deals (ID, DB, LID) VALUES (?, TRUE, FALSE) ON CONFLICT(ID) DO NOTHING`, deal.DealUuid.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark deal migrated: %w", deal.DealUuid.String(), err)
+ }
+ }
+
+ if !b {
+ // Add LID details to pieceDeal in HarmonyDB
+ _, err = tx.Exec(`SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
+ deal.DealUuid.String(), prop.PieceCID.String(), true, mid, deal.SectorID, deal.Offset,
+ prop.PieceSize, deal.NBytesReceived, false, false, deal.ChainDealID)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to update piece metadata and piece deal: %w", deal.DealUuid.String(), err)
+ }
+
+ // Mark deal added to pieceDeal in HarmonyDB
+ _, err = mdb.Exec(`UPDATE Deals SET LID = TRUE WHERE ID = ?`, deal.DealUuid.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark deal LID migrated: %w", deal.DealUuid.String(), err)
+ }
+ }
+
+ if !c {
+ // Check if we can index and announce i.e. we have unsealed copy
+ var exists bool
+ err = tx.QueryRow(`SELECT EXISTS (SELECT 1 FROM sector_location WHERE miner_id = $1
+ AND sector_num = $2
+ AND sector_filetype = 1);`, mid, deal.SectorID).Scan(&exists)
+ if err != nil {
+ return false, fmt.Errorf("seal: %s: failed to check if sector is unsealed: %w", deal.DealUuid.String(), err)
+ }
+
+ if exists {
+ var proof abi.RegisteredSealProof
+ err = tx.QueryRow(`SELECT reg_seal_proof FROM sectors_meta WHERE sp_id = $1 AND sector_num = $2`, mid, deal.SectorID).Scan(&proof)
+ if err != nil {
+ llog.Errorw("failed to get sector proof", "error", err, "deal", deal.DealUuid.String(), "sector", deal.SectorID, "miner", mid)
+ return false, nil
+ }
+
+ // Add deal to mk12 pipeline in Curio for indexing and announcement
+ _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline_migration (
+ uuid, sp_id, piece_cid, piece_size, raw_size, sector, reg_seal_proof, sector_offset, should_announce
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (uuid) DO NOTHING`,
+ deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.NBytesReceived,
+ deal.SectorID, proof, deal.Offset, deal.AnnounceToIPNI)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add deal to pipeline for indexing and announcing: %w", deal.DealUuid.String(), err)
+ }
+ } else {
+ llog.Infof("Skipping indexing as sector %d is not unsealed", deal.SectorID)
+ }
+ }
+ return true, nil
+ }, harmonydb.OptionRetry())
+ if err != nil {
+ return err
+ }
+
+ }
+
+ return nil
+}
+
+func migrateLegacyDeals(ctx context.Context, full v1api.FullNode, activeSectors bitfield.BitField, maddr address.Address, hdb *harmonydb.DB, ds datastore.Batching, mdb *sql.DB) error {
+ mid, err := address.IDFromAddress(maddr)
+ if err != nil {
+ return fmt.Errorf("address.IDFromAddress: %s", err)
+ }
+
+ // Get the deals FSM
+ provDS := namespace.Wrap(ds, datastore.NewKey("/deals/provider"))
+ deals, migrate, err := vfsm.NewVersionedFSM(provDS, fsm.Parameters{
+ StateType: legacytypes.MinerDeal{},
+ StateKeyField: "State",
+ }, nil, "2")
+ if err != nil {
+ return fmt.Errorf("reading legacy deals from datastore: %w", err)
+ }
+
+ err = migrate(ctx)
+ if err != nil {
+ return fmt.Errorf("running provider fsm migration script: %w", err)
+ }
+
+ lm := legacy.NewLegacyDealsManager(deals)
+ go lm.Run(ctx)
+ // Wait for 5 seconds
+ time.Sleep(time.Second * 5)
+
+ legacyDeals, err := lm.ListDeals()
+ if err != nil {
+ return fmt.Errorf("getting legacy deals: %w", err)
+ }
+
+ head, err := full.ChainHead(ctx)
+ if err != nil {
+ return err
+ }
+
+ for i, deal := range legacyDeals {
+ if i > 0 && i%100 == 0 {
+ fmt.Printf("Migrating Legacy Deals: %d / %d (%0.2f%%)\n", i, len(legacyDeals), float64(i)/float64(len(legacyDeals))*100)
+ }
+ llog := log.With("Legacy Deal", deal.ProposalCid.String())
+ // Skip deals which do not have chain deal ID
+ if deal.DealID == 0 {
+ llog.Infow("Skipping as chain deal ID is 0")
+ continue
+ }
+
+ // SKip sector 0. This might cause some deals to not migrate but
+ // that is better than migrating faulty deals
+ if deal.SectorNumber == 0 {
+ llog.Infow("Skipping as sector ID is 0")
+ continue
+ }
+
+ // Skip expired legacy deals
+ if deal.Proposal.EndEpoch < head.Height() {
+ llog.Infow("Deal end epoch is lower than current height")
+ continue
+ }
+
+ // Skip if the sector for the deal is not alive
+ ok, err := activeSectors.IsSet(uint64(deal.SectorNumber))
+ if err != nil {
+ return err
+ }
+ if !ok {
+ llog.Infof("Skipping as sector %d is not alive anymore", deal.SectorNumber)
+ continue
+ }
+
+ // Skip if already migrated
+ var a, b, c bool
+ err = mdb.QueryRow(`SELECT DB, LID, Pipeline FROM Deals WHERE ID = ?`, deal.ProposalCid.String()).Scan(&a, &b, &c)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("deal: %s: failed to check migration status: %w", deal.ProposalCid.String(), err)
+ }
+ }
+ if a && b && c {
+ llog.Infow("Skipped as this deal is already migrated")
+ continue
+ }
+
+ propJson, err := json.Marshal(deal.Proposal)
+ if err != nil {
+ return fmt.Errorf("deal: %s: json.Marshal(piece.DealProposal): %s", deal.ProposalCid.String(), err)
+ }
+
+ sigByte, err := deal.ClientSignature.MarshalBinary()
+ if err != nil {
+ return fmt.Errorf("deal: %s: marshal client signature: %s", deal.ProposalCid.String(), err)
+ }
+
+ prop := deal.Proposal
+
+ _, err = hdb.Exec(ctx, `INSERT INTO market_legacy_deals (signed_proposal_cid, sp_id, client_peer_id,
+ proposal_signature, proposal, piece_cid,
+ piece_size, verified, start_epoch, end_epoch,
+ publish_cid, chain_deal_id, fast_retrieval, created_at, sector_num)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
+ ON CONFLICT (sp_id, piece_cid, signed_proposal_cid) DO NOTHING`,
+ deal.ProposalCid, mid, deal.Client.String(), sigByte, propJson, prop.PieceCID.String(),
+ prop.PieceSize, prop.VerifiedDeal, prop.StartEpoch, prop.EndEpoch, deal.PublishCid.String(),
+ deal.DealID, deal.FastRetrieval, deal.CreationTime.Time(), deal.SectorNumber)
+
+ if err != nil {
+ return fmt.Errorf("deal: %s: failed to add the legacy deal to harmonyDB: %w", deal.ProposalCid.String(), err)
+ }
+
+ // Mark deal added to harmonyDB
+ _, err = mdb.Exec(`INSERT INTO Deals (ID, DB, LID) VALUES (?, TRUE, TRUE) ON CONFLICT(ID) DO NOTHING`, deal.ProposalCid.String())
+ if err != nil {
+ return fmt.Errorf("deal: %s: failed to mark deal migrated: %w", deal.ProposalCid.String(), err)
+ }
+ }
+
+ return nil
+}
+
+func migrateDDODeals(ctx context.Context, full v1api.FullNode, activeSectors bitfield.BitField, maddr address.Address, hdb *harmonydb.DB, sqldb, mdb *sql.DB) error {
+ ddb := db.NewDirectDealsDB(sqldb)
+
+ mid, err := address.IDFromAddress(maddr)
+ if err != nil {
+ return fmt.Errorf("address.IDFromAddress: %s", err)
+ }
+
+ deals, err := ddb.ListAll(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to get all DDO deals: %w", err)
+ }
+
+ for i, deal := range deals {
+ if i > 0 && i%100 == 0 {
+ fmt.Printf("Migrating DDO Deals: %d / %d (%0.2f%%)\n", i, len(deals), float64(i)/float64(len(deals))*100)
+ }
+ llog := log.With("DDO Deal", deal.ID.String())
+ if deal.Err != "" && deal.Retry == types.DealRetryFatal {
+ llog.Infow("Skipping as deal retry is fatal")
+ continue
+ }
+
+ if deal.Checkpoint < dealcheckpoints.AddedPiece {
+ llog.Infow("Skipping as checkpoint is below add piece")
+ continue
+ }
+
+ claim, err := full.StateGetClaim(ctx, maddr, verifreg9types.ClaimId(deal.AllocationID), ltypes.EmptyTSK)
+ if err != nil {
+ return fmt.Errorf("deal: %s: error getting the claim status: %w", deal.ID.String(), err)
+ }
+ if claim == nil {
+ llog.Infow("Skipping as checkpoint is below add piece")
+ continue
+ }
+ if claim.Sector != deal.SectorID {
+ return fmt.Errorf("deal: %s: sector mismatch for deal", deal.ID.String())
+ }
+
+ // Skip if the sector for the deal is not alive
+ ok, err := activeSectors.IsSet(uint64(deal.SectorID))
+ if err != nil {
+ return err
+ }
+ if !ok {
+ llog.Infow("Skipping as sector ID is 0")
+ continue
+ }
+
+ // Skip if already migrated
+ var a, b, c bool
+ err = mdb.QueryRow(`SELECT DB, LID, Pipeline FROM Deals WHERE ID = ?`, deal.ID.String()).Scan(&a, &b, &c)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("deal: %s: failed to check migration status: %w", deal.ID.String(), err)
+ }
+ }
+ if a && b && c {
+ continue
+ }
+
+ _, err = hdb.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) {
+ if !a {
+ // Add DDO deal to harmonyDB
+ _, err = tx.Exec(`INSERT INTO market_direct_deals (uuid, sp_id, created_at, client, offline, verified,
+ start_epoch, end_epoch, allocation_id, piece_cid, piece_size, fast_retrieval, announce_to_ipni)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+ ON CONFLICT (uuid) DO NOTHING`,
+ deal.ID.String(), mid, deal.CreatedAt, deal.Client.String(), true, true, deal.StartEpoch, deal.EndEpoch, deal.AllocationID,
+ deal.PieceCID.String(), deal.PieceSize, true, true)
+
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add the DDO deal to harmonyDB: %w", deal.ID.String(), err)
+ }
+
+ // Mark deal added to harmonyDB
+ _, err = mdb.Exec(`INSERT INTO Deals (ID, DB, LID) VALUES (?, TRUE, FALSE) ON CONFLICT(ID) DO NOTHING`, deal.ID.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark DDO deal migrated: %w", deal.ID.String(), err)
+ }
+ }
+
+ if !b {
+ // Add LID details to pieceDeal in HarmonyDB
+ _, err = tx.Exec(`SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
+ deal.ID.String(), deal.PieceCID.String(), false, mid, deal.SectorID, deal.Offset, deal.PieceSize, deal.InboundFileSize, false)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to update piece metadata and piece deal for DDO deal %s: %w", deal.ID.String(), deal.ID.String(), err)
+ }
+
+ // Mark deal added to pieceDeal in HarmonyDB
+ _, err = mdb.Exec(`UPDATE Deals SET LID = TRUE WHERE ID = ?`, deal.ID.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark deal LID migrated: %w", deal.ID.String(), err)
+ }
+ }
+
+ // TODO: Confirm if using the mk12 pipeline will have any impact for DDO deals
+ if !c {
+ // Check if we can index and announce i.e. we have unsealed copy
+ var exists bool
+ err = tx.QueryRow(`SELECT EXISTS (SELECT 1 FROM sector_location WHERE miner_id = $1
+ AND sector_num = $2
+ AND sector_filetype = 1);`, mid, deal.SectorID).Scan(&exists)
+ if err != nil {
+ return false, fmt.Errorf("seal: %s: failed to check if sector is unsealed: %w", deal.ID.String(), err)
+ }
+
+ if exists {
+ var proof abi.RegisteredSealProof
+ err = tx.QueryRow(`SELECT reg_seal_proof FROM sectors_meta WHERE sp_id = $1 AND sector_num = $2`, mid, deal.SectorID).Scan(&proof)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to get sector proof: %w", deal.ID.String(), err)
+ }
+
+ // Add deal to mk12 pipeline in Curio for indexing and announcement
+ _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline_migration (
+ uuid, sp_id, piece_cid, piece_size, raw_size, sector, reg_seal_proof, sector_offset, should_announce
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (uuid) DO NOTHING`,
+ deal.ID.String(), mid, deal.PieceCID.String(), deal.PieceSize, deal.InboundFileSize,
+ deal.SectorID, proof, deal.Offset, true)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add DDO deal to pipeline for indexing and announcing: %w", deal.ID.String(), err)
+ }
+ } else {
+ llog.Infof("Skipping indexing as sector %d is not unsealed", deal.SectorID)
+ }
+ }
+ return true, nil
+ }, harmonydb.OptionRetry())
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/cmd/migrate-curio/yugabyte.go b/cmd/migrate-curio/yugabyte.go
new file mode 100644
index 000000000..b90b058b7
--- /dev/null
+++ b/cmd/migrate-curio/yugabyte.go
@@ -0,0 +1,139 @@
+package main
+
+import (
+ "fmt"
+
+ "github.com/filecoin-project/boost/extern/boostd-data/yugabyte"
+ "github.com/urfave/cli/v2"
+ "github.com/yugabyte/pgx/v4/pgxpool"
+)
+
+var cleanupLIDCmd = &cli.Command{
+ Name: "cleanup",
+ Description: "Removes the indexes and other metadata from LID",
+ Usage: "migrate-curio cleanup",
+ Subcommands: []*cli.Command{
+ cleanupLevelDBCmd,
+ cleanupYugabyteDBCmd,
+ },
+}
+
+var cleanupYugabyteDBCmd = &cli.Command{
+ Name: "yugabyte",
+ Description: "Removes the indexes and other metadata from Yugabyte based LID store",
+ Usage: "migrate-curio cleanup yugabyte",
+ Before: before,
+ Flags: []cli.Flag{
+ &cli.StringSliceFlag{
+ Name: "hosts",
+ Usage: "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'",
+ Required: true,
+ },
+ &cli.StringFlag{
+ Name: "username",
+ Usage: "yugabyte username to connect to over cassandra interface eg 'cassandra'",
+ },
+ &cli.StringFlag{
+ Name: "password",
+ Usage: "yugabyte password to connect to over cassandra interface eg 'cassandra'",
+ },
+ &cli.StringFlag{
+ Name: "connect-string",
+ Usage: "postgres connect string eg 'postgresql://postgres:postgres@localhost'",
+ Required: true,
+ },
+ &cli.BoolFlag{
+ Name: "i-know-what-i-am-doing",
+ Usage: "confirmation flag",
+ Value: false,
+ },
+ },
+ Action: func(cctx *cli.Context) error {
+ if !cctx.Bool("i-know-what-i-am-doing") {
+ return fmt.Errorf("please use --i-know-what-i-am-doing flag to confirm. THIS CANNOT BE UNDONE")
+ }
+
+ // Create a yugabyte data service
+ settings := yugabyte.DBSettings{
+ Hosts: cctx.StringSlice("hosts"),
+ Username: cctx.String("username"),
+ Password: cctx.String("password"),
+ ConnectString: cctx.String("connect-string"),
+ }
+
+ settings.CQLTimeout = 60
+
+ cluster := yugabyte.NewCluster(settings)
+
+ session, err := cluster.CreateSession()
+ if err != nil {
+ return fmt.Errorf("creating yugabyte cluster: %w", err)
+ }
+
+ keyspace := "idx"
+
+ // Drop all indexes
+ var indexName string
+ indexesQuery := fmt.Sprintf("SELECT index_name FROM system_schema.indexes WHERE keyspace_name='%s';", keyspace)
+ iter := session.Query(indexesQuery).Iter()
+
+ for iter.Scan(&indexName) {
+ dropIndexQuery := fmt.Sprintf("DROP INDEX %s.%s;", keyspace, indexName)
+ fmt.Println("Executing:", dropIndexQuery)
+ if err := session.Query(dropIndexQuery).Exec(); err != nil {
+ return fmt.Errorf("failed to drop index %s: %w", indexName, err)
+ }
+ }
+ if err := iter.Close(); err != nil {
+ return fmt.Errorf("failed to iterate over indexes: %w", err)
+ }
+
+ // Query to get all tables in the 'idx' keyspace
+ tableQuesry := fmt.Sprintf(`SELECT table_name FROM system_schema.tables WHERE keyspace_name='%s';`, keyspace)
+ iter = session.Query(tableQuesry).Iter()
+
+ var tableName string
+ for iter.Scan(&tableName) {
+ dropQuery := fmt.Sprintf("DROP TABLE idx.%s", tableName)
+ fmt.Println("Executing:", dropQuery)
+
+ err := session.Query(dropQuery).Exec()
+ if err != nil {
+ return fmt.Errorf("failed to drop table %s: %w", tableName, err)
+ }
+ }
+
+ if err := iter.Close(); err != nil {
+ return fmt.Errorf("error closing iterator: %v", err)
+ }
+
+ fmt.Println("All tables in keyspace 'idx' have been dropped.")
+
+ // Drop the keyspace
+ dropKeyspaceQuery := fmt.Sprintf("DROP KEYSPACE %s;", keyspace)
+ fmt.Println("Executing:", dropKeyspaceQuery)
+ if err := session.Query(dropKeyspaceQuery).Exec(); err != nil {
+ return fmt.Errorf("failed to drop keyspace: %w", err)
+ }
+
+ fmt.Println("Keyspace dropped successfully.")
+
+ // Create connection pool to postgres interface
+ db, err := pgxpool.Connect(cctx.Context, settings.ConnectString)
+ if err != nil {
+ return err
+ }
+
+ _, err = db.Exec(cctx.Context, `DROP TABLE IF EXISTS PieceTracker CASCADE;`)
+ if err != nil {
+ return err
+ }
+
+ _, err = db.Exec(cctx.Context, `DROP TABLE IF EXISTS PieceFlagged CASCADE;`)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ },
+}
diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md
index 859bfc871..5dfd09678 100644
--- a/documentation/en/api-v1-methods.md
+++ b/documentation/en/api-v1-methods.md
@@ -20,6 +20,7 @@
* [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
* [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal)
* [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
+ * [BoostIndexerRemoveAll](#boostindexerremoveall)
* [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid)
* [BoostOfflineDealWithData](#boostofflinedealwithdata)
* [I](#i)
@@ -374,7 +375,7 @@ Response:
```
### BoostIndexerAnnounceAllDeals
-There are not yet any comments for this method.
+
Perms: admin
@@ -513,6 +514,20 @@ Response:
]
```
+### BoostIndexerRemoveAll
+There are not yet any comments for this method.
+
+Perms: admin
+
+Inputs: `null`
+
+Response:
+```json
+[
+ null
+]
+```
+
### BoostLegacyDealByProposalCid
diff --git a/go.mod b/go.mod
index 531954438..f16a47680 100644
--- a/go.mod
+++ b/go.mod
@@ -17,10 +17,11 @@ require (
github.com/docker/go-units v0.5.0
github.com/dustin/go-humanize v1.0.1
github.com/etclabscore/go-openrpc-reflect v0.0.37
- github.com/ethereum/go-ethereum v1.13.15
+ github.com/ethereum/go-ethereum v1.14.13
github.com/fatih/color v1.18.0
github.com/filecoin-project/boost-graphsync v0.13.12
github.com/filecoin-project/boost/extern/boostd-data v0.0.0-20231124125934-3233c510357f
+ github.com/filecoin-project/curio v1.25.0
github.com/filecoin-project/dagstore v0.7.0
github.com/filecoin-project/go-address v1.2.0
github.com/filecoin-project/go-bitfield v0.2.4
@@ -108,6 +109,7 @@ require (
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11
github.com/whyrusleeping/cbor-gen v0.3.1
+ github.com/yugabyte/pgx/v4 v4.14.5
go.opencensus.io v0.24.0
go.opentelemetry.io/otel v1.35.0
go.opentelemetry.io/otel/trace v1.35.0
@@ -143,7 +145,6 @@ require (
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
- github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -177,6 +178,7 @@ require (
github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect
github.com/filecoin-project/go-amt-ipld/v4 v4.4.0 // indirect
github.com/filecoin-project/go-clock v0.1.0 // indirect
+ github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 // indirect
github.com/filecoin-project/go-commp-utils/v2 v2.1.0 // indirect
github.com/filecoin-project/go-crypto v0.1.0 // indirect
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc8 // indirect
@@ -262,7 +264,7 @@ require (
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jessevdk/go-flags v1.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
- github.com/kilic/bls12-381 v0.1.0 // indirect
+ github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/koron/go-ssdp v0.0.6 // indirect
@@ -340,6 +342,7 @@ require (
github.com/samber/lo v1.47.0 // indirect
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
+ github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -356,8 +359,7 @@ require (
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
- github.com/yugabyte/gocql v0.0.0-20230831121436-1e2272bb6bb6 // indirect
- github.com/yugabyte/pgx/v4 v4.14.5 // indirect
+ github.com/yugabyte/gocql v1.6.0-yb-1 // indirect
github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // indirect
github.com/zondax/hid v0.9.2 // indirect
github.com/zondax/ledger-filecoin-go v1.1.0 // indirect
diff --git a/go.sum b/go.sum
index faa1dae33..721655de7 100644
--- a/go.sum
+++ b/go.sum
@@ -143,8 +143,6 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurT
github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
-github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ=
-github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
@@ -305,8 +303,8 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakR
github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ=
github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c=
github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E=
-github.com/ethereum/go-ethereum v1.13.15 h1:U7sSGYGo4SPjP6iNIifNoyIAiNjrmQkz6EwQG+/EZWo=
-github.com/ethereum/go-ethereum v1.13.15/go.mod h1:TN8ZiHrdJwSe8Cb6x+p0hs5CxhJZPbqB7hHkaUXcmIU=
+github.com/ethereum/go-ethereum v1.14.13 h1:L81Wmv0OUP6cf4CW6wtXsr23RUrDhKs2+Y9Qto+OgHU=
+github.com/ethereum/go-ethereum v1.14.13/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
@@ -317,6 +315,8 @@ github.com/filecoin-project/boost-graphsync v0.13.12 h1:fAGaHRIYoN6cPMs2ChVymio8
github.com/filecoin-project/boost-graphsync v0.13.12/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk=
github.com/filecoin-project/boost/extern/boostd-data v0.0.0-20231124125934-3233c510357f h1:8dd0yAadyeOL5Qd42XhEwD60UKvIFkY2MLhef/IaeOk=
github.com/filecoin-project/boost/extern/boostd-data v0.0.0-20231124125934-3233c510357f/go.mod h1:MyzvfYWAH0OAyf95TLUWYq3cO3vm/TVzDS57GKQi47o=
+github.com/filecoin-project/curio v1.25.0 h1:rbAkHzcYzYSRuTtX1bY/2lMzhdHFtP37zy6VZDs+3eI=
+github.com/filecoin-project/curio v1.25.0/go.mod h1:bAQYgPNO+JgSrxONFVfrEYEZ0xzpQRFmOpCUg7anQGc=
github.com/filecoin-project/dagstore v0.7.0 h1:IS0R+69za8dguYWeqz/MI+nb7ONpk03tAkxPCBXEKm0=
github.com/filecoin-project/dagstore v0.7.0/go.mod h1:YKn4qXih+/2xQWpfJsaKGOi4POw5vH5grDmfPCCnx8g=
github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
@@ -342,6 +342,8 @@ github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9g
github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs=
github.com/filecoin-project/go-commp-utils v0.1.4 h1:/WSsrAb0xupo+aRWRyD80lRUXAXJvYoTgDQS1pYZ1Mk=
github.com/filecoin-project/go-commp-utils v0.1.4/go.mod h1:Sekocu5q9b4ECAUFu853GFUbm8I7upAluummHFe2kFo=
+github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 h1:jAG2g1Fs/qoDSSaI8JaP/KmqR+QQ8IVQ6k9xKONa72M=
+github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8/go.mod h1:kU2KuSPLB+Xz4FEbVE0abzSN4l6irZ8tqgcYWPVDftU=
github.com/filecoin-project/go-commp-utils/v2 v2.1.0 h1:KWNRalUp2bhN1SW7STsJS2AHs9mnfGKk9LnQgzDe+gI=
github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj/gysLNu26kYII83IV5iNrAO9iI=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
@@ -1037,8 +1039,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
-github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4=
-github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig=
+github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 h1:xWK4TZ4bRL05WQUU/3x6TG1l+IYAqdXpAeSLt/zZJc4=
+github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4/go.mod h1:tlkavyke+Ac7h8R3gZIjI5LKBcvMlSWnXNMgT3vZXo8=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -1764,6 +1766,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
+github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb h1:78YgPq3NbWnO4xyNhLsn2zitc7NiZpjQZ560rsxVLm4=
+github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb/go.mod h1:YnCEHk4UlWROjM3YzC2pbSTq+iynM3ZaLoVDUI8QGpE=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
@@ -1901,8 +1905,8 @@ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZM
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
-github.com/yugabyte/gocql v0.0.0-20230831121436-1e2272bb6bb6 h1:3sggdexlJg07v9sRBKkZ+/oSJ0sq53vVApx8iSbD98M=
-github.com/yugabyte/gocql v0.0.0-20230831121436-1e2272bb6bb6/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M=
+github.com/yugabyte/gocql v1.6.0-yb-1 h1:3anNiHsJwKQ8Dn7RdmkTEuIzV1l7e9QJZ8wkOZ87ELg=
+github.com/yugabyte/gocql v1.6.0-yb-1/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M=
github.com/yugabyte/pgx/v4 v4.14.5 h1:XLzEEiO3d/kWzpyctO8l4kwMLhzPQ9n2er7ATH7CJVA=
github.com/yugabyte/pgx/v4 v4.14.5/go.mod h1:nFSvjsVq4CuA61TWGriwWG74ZVxeuJCcNu42Mkn+rgw=
github.com/yugabyte/pgx/v5 v5.5.3-yb-2 h1:SDk2waZb2o6dSLYqk+vq0Ur2jnIv+X2A+P+QPR1UThU=
@@ -2276,7 +2280,6 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/go.work.sum b/go.work.sum
index 3c161e4c8..7ebe1e3e4 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -1853,6 +1853,8 @@ github.com/BurntSushi/toml v1.3.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
+github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg=
+github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM=
github.com/ClickHouse/ch-go v0.52.1/go.mod h1:B9htMJ0hii/zrC2hljUKdnagRBuLqtRG/GrU3jqCwRk=
github.com/ClickHouse/ch-go v0.57.0 h1:X/QmUmFhpUvLgPSQb7fWOSi1wvqGn6tJ7w2a59c4xsg=
github.com/ClickHouse/ch-go v0.57.0/go.mod h1:DR3iBn7OrrDj+KeUp1LbdxLEUDbW+5Qwdl/qkc+PQ+Y=
@@ -1903,6 +1905,8 @@ github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7 h1:mreN1m/5VJ/Zc3b4pz
github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk=
github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM=
+github.com/KarpelesLab/reflink v1.0.1 h1:d+tdjliwOCqvub9bl0Y02GxahWkNqejNb3TZTTUcQWA=
+github.com/KarpelesLab/reflink v1.0.1/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk=
github.com/Masterminds/glide v0.13.2 h1:M5MOH04TyRiMBVeWHbifqTpnauxWINIubTCOkhXh+2g=
@@ -1977,6 +1981,7 @@ github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c
github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40=
github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
+github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
@@ -2197,6 +2202,8 @@ github.com/btcsuite/btcd v0.23.5-0.20231215221805-96c9fd8078fd/go.mod h1:nm3Bko6
github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA=
github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A=
github.com/btcsuite/btcd/btcutil v1.1.0/go.mod h1:5OapHB7A2hBBWLm48mmw4MOHNJCcUBTwmWH/0Jn8VHE=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
@@ -2327,6 +2334,7 @@ github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lg
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A=
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo=
github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
+github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
github.com/cockroachdb/pebble v1.1.4 h1:5II1uEP4MyHLDnsrbv/EZ36arcb9Mxg3n+owhZ3GrG8=
github.com/cockroachdb/pebble v1.1.4/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
@@ -2481,9 +2489,11 @@ github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O
github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
+github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I=
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc=
+github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI=
github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc=
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
@@ -2501,6 +2511,7 @@ github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2
github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI=
github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
+github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM=
github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
@@ -2572,11 +2583,13 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNE
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao=
github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw=
github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 h1:kgvzE5wLsLa7XKfV85VZl40QXaMCaeFtHpPwJ8fhotY=
github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
+github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE=
github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5RVoRccG8a5DhOdWdQ4zN62zzo=
github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
@@ -2585,6 +2598,8 @@ github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d h1:W1n4DvpzZGOI
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE=
github.com/drand/drand v1.3.0 h1:k/w/PtHzmlU6OmfoAqgirWyrJ4FZH8ESlJrsKF20UkM=
+github.com/drand/drand v1.5.11 h1:7sskUTCsX2lgFiWdGvPh3/P0ZDQKi1lCtI7RQKK010k=
+github.com/drand/drand v1.5.11/go.mod h1:TvJjCJ/s4Usn4pKRpDC0N1QaCwSt3YC8fRqhZdpOUU0=
github.com/drand/kyber v1.1.10/go.mod h1:UkHLsI4W6+jT5PvNxmc0cvQAgppjTUpX+XCsN9TXmRo=
github.com/drand/kyber v1.1.17/go.mod h1:2SbJSUoZt8D61uMWH2QTEWqzK9BBYzcObeb1TKDKwvo=
github.com/drand/kyber v1.2.0/go.mod h1:6TqFlCc7NGOiNVTF9pF2KcDRfllPd9XOkExuG5Xtwfo=
@@ -2667,9 +2682,11 @@ github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY=
github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
+github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA=
github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
github.com/ethereum/go-ethereum v1.10.26/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg=
github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w=
+github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A=
github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk=
github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q=
github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A=
@@ -3007,6 +3024,8 @@ github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
+github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
@@ -3640,6 +3659,7 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3 h1:Iy7Ifq2ysilWU4QlCx/97OoI4xT1IV7i8byT/EyIT/M=
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d h1:cVtBfNW5XTHiKQe7jDaDBSh/EVM4XLPutLAGboIXuM0=
+github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 h1:msKODTL1m0wigztaqILOtla9HeW1ciscYG4xjLtvk5I=
github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8=
github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4=
github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
@@ -3655,6 +3675,7 @@ github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4
github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=
github.com/kataras/golog v0.1.7 h1:0TY5tHn5L5DlRIikepcaRR/6oInIr9AiWsxzt0vvlBE=
github.com/kataras/golog v0.1.7/go.mod h1:jOSQ+C5fUqsNSwurB/oAHq1IFSb0KI3l6GMa7xB6dZA=
+github.com/kataras/golog v0.1.8 h1:isP8th4PJH2SrbkciKnylaND9xoTtfxv++NB+DF0l9g=
github.com/kataras/golog v0.1.8/go.mod h1:rGPAin4hYROfk1qT9wZP6VY2rsb4zzc37QpdPjdkqVw=
github.com/kataras/golog v0.1.11 h1:dGkcCVsIpqiAMWTlebn/ZULHxFvfG4K43LF1cNWSh20=
github.com/kataras/golog v0.1.11/go.mod h1:mAkt1vbPowFUuUGvexyQ5NFW6djEgGyxQBIARJ0AH4A=
@@ -3663,6 +3684,7 @@ github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/l
github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=
github.com/kataras/iris/v12 v12.2.0-beta5 h1:grB/oCf5baZhmYIeDMfgN3LYrtEcmK8pbxlRvEZ2pgw=
github.com/kataras/iris/v12 v12.2.0-beta5/go.mod h1:q26aoWJ0Knx/00iPKg5iizDK7oQQSPjbD8np0XDh6dc=
+github.com/kataras/iris/v12 v12.2.0 h1:WzDY5nGuW/LgVaFS5BtTkW3crdSKJ/FEgWnxPnIVVLI=
github.com/kataras/iris/v12 v12.2.0/go.mod h1:BLzBpEunc41GbE68OUaQlqX4jzi791mx5HU04uPb90Y=
github.com/kataras/iris/v12 v12.2.10 h1:rEJVM7qMoyhv8wpgkA1yGxibFcONE0jkJ70LFLibTAA=
github.com/kataras/iris/v12 v12.2.10/go.mod h1:z4+E+kLMqZ7U4WtDsYfFnG7BjMTXLkdzMAXLVMLnMNs=
@@ -3692,6 +3714,7 @@ github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA=
github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw=
github.com/keybase/go-crypto v0.0.0-20180614160407-5114a9a81e1b h1:VE6r2OwP5gj+Z9aCkSKl3MlmnZbfMAjhvR5T7abKHEo=
github.com/keybase/go-crypto v0.0.0-20180614160407-5114a9a81e1b/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M=
+github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig=
github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY=
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
@@ -3752,6 +3775,7 @@ github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awS
github.com/labstack/echo/v4 v4.7.2/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks=
github.com/labstack/echo/v4 v4.9.0 h1:wPOF1CE6gvt/kmbMR4dGzWvHMPT+sAEUJOwOTtvITVY=
github.com/labstack/echo/v4 v4.9.0/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks=
+github.com/labstack/echo/v4 v4.10.0 h1:5CiyngihEO4HXsz3vVsJn7f8xAlWwRr3aY6Ih280ZKA=
github.com/labstack/echo/v4 v4.10.0/go.mod h1:S/T/5fy/GigaXnHTkh0ZGe4LpkkQysvRjFMSUTkDRNQ=
github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
@@ -3759,6 +3783,7 @@ github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o=
github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
+github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8=
github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
@@ -4041,6 +4066,7 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50=
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
+github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
@@ -4573,9 +4599,13 @@ github.com/prometheus/statsd_exporter v0.26.1/go.mod h1:XlDdjAmRmx3JVvPPYuFNUg+Y
github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 h1:cZC+usqsYgHtlBaGulVnZ1hfKAi8iWtujBnRLQE698c=
github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY=
+github.com/protolambda/bls12-381-util v0.1.0 h1:05DU2wJN7DTU7z28+Q+zejXkIsA/MF8JZQGhtBZZiWk=
github.com/protolambda/bls12-381-util v0.1.0/go.mod h1:cdkysJTRpeFeuUVx/TXGDQNMTiRAalk1vQw3TYTHcE4=
+github.com/protolambda/messagediff v1.4.0 h1:fk6gxK7WybJCaeOFK1yuh2Ldplx7qYMLibiMwWFcSZY=
github.com/protolambda/messagediff v1.4.0/go.mod h1:LboJp0EwIbJsePYpzh5Op/9G1/4mIztMRYzzwR0dR2M=
+github.com/protolambda/zrnt v0.32.2 h1:KZ48T+3UhsPXNdtE/5QEvGc9DGjUaRI17nJaoznoIaM=
github.com/protolambda/zrnt v0.32.2/go.mod h1:A0fezkp9Tt3GBLATSPIbuY4ywYESyAuc/FFmPKg8Lqs=
+github.com/protolambda/ztyp v0.2.2 h1:rVcL3vBu9W/aV646zF6caLS/dyn9BN8NYiuJzicLNyY=
github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU=
github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 h1:cSo6/vk8YpvkLbk9v3FO97cakNmUoxwi2KMP8hd5WIw=
github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48/go.mod h1:4pWaT30XoEx1j8KNJf3TV+E3mQkaufn7mf+jRNb/Fuk=
@@ -4818,6 +4848,7 @@ github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 h1:m+8fKfQwCA
github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
+github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk=
github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -5324,12 +5355,12 @@ golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
-golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
@@ -5594,6 +5625,7 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -5663,10 +5695,10 @@ golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808 h1:+Kc94D8UVEVxJnLXp/+FMfqQARZtWHfVrcRtcG8aT3g=
@@ -6243,6 +6275,7 @@ gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go
index 0976ad1a2..0b564bee4 100644
--- a/indexprovider/wrapper.go
+++ b/indexprovider/wrapper.go
@@ -18,11 +18,11 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
chainTypes "github.com/filecoin-project/lotus/chain/types"
"github.com/google/uuid"
- cbor "github.com/ipfs/go-ipld-cbor"
- "go.uber.org/fx"
-
"github.com/ipfs/go-datastore"
+ cbor "github.com/ipfs/go-ipld-cbor"
"github.com/ipld/go-ipld-prime"
+ "github.com/ipni/go-libipni/ingest/schema"
+ "go.uber.org/fx"
"github.com/filecoin-project/boost/db"
bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types"
@@ -867,3 +867,52 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u
}
return annCid, err
}
+
+func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+ var allAds []*schema.Advertisement
+ _, ad, err := w.prov.GetLatestAdv(ctx)
+ if err != nil {
+ return nil, err
+ }
+ allAds = append(allAds, ad)
+
+ prev, err := cid.Parse(ad.PreviousID.String())
+ if err != nil {
+ return nil, err
+ }
+
+ for prev != cid.Undef {
+ ad, err := w.prov.GetAdv(ctx, prev)
+ if err != nil {
+ return nil, err
+ }
+
+ prev, err = cid.Parse(ad.PreviousID.String())
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ var entryAds []*schema.Advertisement
+
+ for _, ad := range allAds {
+ if !ad.IsRm {
+ entryAds = append(entryAds, ad)
+ }
+ }
+
+ var newAds []cid.Cid
+
+ for _, ad := range entryAds {
+ a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID)
+ if err != nil {
+ if !errors.Is(err, provider.ErrContextIDNotFound) {
+ return nil, fmt.Errorf("failed to publish the removal ad: %w", err)
+ }
+ }
+ newAds = append(newAds, a)
+ }
+
+ return newAds, nil
+
+}
diff --git a/itests/framework/framework.go b/itests/framework/framework.go
index e916c1a3a..2a72e6920 100644
--- a/itests/framework/framework.go
+++ b/itests/framework/framework.go
@@ -401,6 +401,12 @@ func (f *TestFramework) Start(opts ...ConfigOpt) error {
// Enable LID with leveldb
cfg.LocalIndexDirectory.Leveldb.Enabled = true
+ cfg.IndexProvider.Announce.AnnounceOverHttp = true
+ //cfg.IndexProvider.DataTransferPublisher = false
+ cfg.IndexProvider.HttpPublisher.Enabled = true
+ //cfg.IndexProvider.HttpPublisher.Port = 3333
+ cfg.IndexProvider.HttpPublisher.PublicHostname = "127.0.0.1"
+ cfg.IndexProvider.HttpPublisher.WithLibp2p = true
// Test IPNI config
cfg.IndexProvider.HttpPublisher.WithLibp2p = true
diff --git a/itests/ipni_publish_test.go b/itests/ipni_publish_test.go
index dfdee16f1..2296f7d55 100644
--- a/itests/ipni_publish_test.go
+++ b/itests/ipni_publish_test.go
@@ -6,9 +6,7 @@ import (
"testing"
"github.com/davecgh/go-spew/spew"
- boost_build "github.com/filecoin-project/boost/build"
"github.com/filecoin-project/boost/itests/framework"
- "github.com/filecoin-project/boost/node/modules/dtypes"
"github.com/filecoin-project/boost/testutil"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/itests/kit"
@@ -34,13 +32,8 @@ func TestIPNIPublish(t *testing.T) {
addrs, err := f.Boost.NetAddrsListen(ctx)
require.NoError(t, err)
- ntwkName, err := f.FullNode.StateNetworkName(ctx)
- require.NoError(t, err)
-
- topicName := boost_build.IndexerIngestTopic(dtypes.NetworkName(ntwkName))
-
// Create new ipni-cli client
- ipniClient, err := adpub.NewClient(addrs, adpub.WithTopicName(topicName), adpub.WithEntriesDepthLimit(100000))
+ ipniClient, err := adpub.NewClient(addrs, adpub.WithEntriesDepthLimit(100000))
require.NoError(t, err)
// Get head when boost starts
diff --git a/node/impl/boost.go b/node/impl/boost.go
index 645fb0e49..c27be73ff 100644
--- a/node/impl/boost.go
+++ b/node/impl/boost.go
@@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error {
func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) {
return sm.StorageProvider.GetAsk(), nil
}
+
+func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+ return sm.IndexProvider.AnnounceRemoveAll(ctx)
+}
diff --git a/react/src/Ipni.js b/react/src/Ipni.js
index cd7bc782a..d2b5b610b 100644
--- a/react/src/Ipni.js
+++ b/react/src/Ipni.js
@@ -111,13 +111,14 @@ function ProviderIpniInfoRender(props){
({moment(data.LastAdvertisementTime).fromNow()} ago)
- {distance.data ? ({distance.data.ipniDistanceFromLatestAd} behind): ''}
+ {distance.data ?
+ ({distance.data.ipniDistanceFromLatestAd} behind) : ''}
| Latest Advertisement on Boost |
- {lad ? {lad}: ''}
+ {lad ? {lad} : ''}
|
@@ -137,7 +138,7 @@ function ProviderConfig({configJson}) {
const cfg = JSON.parse(configJson)
return
Index Provider Config
-
+
}