Merge branch 'main' of https://github.com/endomorphosis/ipfs_transfor…

…mers

README.md

@@ -1,42 +1,4 @@
-# IPFS Huggingface Bridge
-
-for transformers.js visit:                          
-https://github.com/endomorphosis/ipfs_transformers_js
-
-for huggingface datasets python library visit:
-https://github.com/endomorphosis/ipfs_datasets
-
-for faiss KNN index python library visit:
-https://github.com/endomorphosis/ipfs_faiss
-
-for orbitdb_kit nodejs library visit:
-https://github.com/endomorphosis/orbitdb_kit/
-
-for python model manager library visit: 
-https://github.com/endomorphosis/ipfs_model_manager/
-
-for nodejs model manager library visit: 
-https://github.com/endomorphosis/ipfs_model_manager_js/
-
-for nodejs ipfs huggingface scraper with pinning services visit:
-https://github.com/endomorphosis/ipfs_huggingface_scraper/
-
-for ipfs huggingface agents visit:
-https://github.com/endomorphosis/ipfs_agents
-
-for ipfs huggingface accelerate visit:
-https://github.com/endomorphosis/ipfs_accelerate
-
-for ipfs_kit visit:
-https://github.com/endomorphosis/ipfs_kit
-
-for ipfs_kit_js visit:
-https://github.com/endomorphosis/ipfs_kit_js
-
-Author - Benjamin Barber
-QA - Kevin De Haan
-
-# About
+# IPFS Huggingface Transformers
 
 This is a model manager and wrapper for huggingface, looks up a index of models from an collection of models, and will download a model from either https/s3/ipfs, depending on which source is the fastest.
 
@@ -79,26 +41,44 @@ model = T5Model.from_auto_download(
 )
 ~~~
 
-# To scrape huggingface
 
-with interactive prompt:
+# IPFS Huggingface Bridge
+
+for transformers.js visit:                          
+https://github.com/endomorphosis/ipfs_transformers_js
 
-~~~shell
-node scraper.js [source] [model name]
-~~~
+for huggingface datasets python library visit:
+https://github.com/endomorphosis/ipfs_datasets
 
-~~~shell
-node scraper.js 
-~~~
+for faiss KNN index python library visit:
+https://github.com/endomorphosis/ipfs_faiss
 
-import a model already defined:
+for orbitdb_kit nodejs library visit:
+https://github.com/endomorphosis/orbitdb_kit/
 
-~~~shell
-node scraper.js hf "modelname" (as defined in your .json files)
-~~~
+for fireproof_kit nodejs library visit:
+https://github.com/endomorphosis/fireproof_kit
 
-import all models previously defined:
+for python model manager library visit: 
+https://github.com/endomorphosis/ipfs_model_manager/
 
-~~~shell
-node scraper.js hf 
-~~~
+for nodejs model manager library visit: 
+https://github.com/endomorphosis/ipfs_model_manager_js/
+
+for nodejs ipfs huggingface scraper with pinning services visit:
+https://github.com/endomorphosis/ipfs_huggingface_scraper/
+
+for ipfs huggingface agents visit:
+https://github.com/endomorphosis/ipfs_agents
+
+for ipfs huggingface accelerate visit:
+https://github.com/endomorphosis/ipfs_accelerate
+
+for ipfs_kit visit:
+https://github.com/endomorphosis/ipfs_kit
+
+for ipfs_kit_js visit:
+https://github.com/endomorphosis/ipfs_kit_js
+
+Author - Benjamin Barber
+QA - Kevin De Haan

example.py

@@ -3,8 +3,8 @@
 
 model = AutoModel.from_auto_download("bge-small-en-v1.5")  
 print(dir(model))
-model = AutoModel.from_ipfs("QmccfbkWLYs9K3yucc6b3eSt8s8fKcyRRt24e3CDaeRhM1")
-print(dir(model))
+# model = AutoModel.from_ipfs("QmccfbkWLYs9K3yucc6b3eSt8s8fKcyRRt24e3CDaeRhM1")
+# print(dir(model))
 
 
 ## OPTIONAL S3 Caching ##