HuggingFace Hub Upload

Upload Language Model


export HF_TOKEN=token


model.push_to_hub(repo_id,
                  use_auth_token=True,
                  # token=token,
                  commit_message=commit_message)

Upload Files


from huggingface_hub import HfApi

hf_api = HfApi(token=token)
hf_api.create_repo(repo_type=repo_type, # dataset | model
                   repo_id=f"{user}/{model}",
                   exist_ok=True)
api.upload_folder(folder_path=f"models/{model}",
                  repo_type=repo_type, # dataset | model
                  repo_id=f"{user}/{model}")

Custom PyTorchModelHubMixin


class CustomPyTorchModelHubMixin(PyTorchModelHubMixin):
    def _save_pretrained(
        self,
        save_directory: Union[str, os.PathLike],
        save_config: bool = True,
        state_dict: Optional[dict] = None,
        save_function: Callable = torch.save,
        push_to_hub: bool = False,
        **kwargs,
    ):
        if os.path.isfile(save_directory):
            logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
            return

        os.makedirs(save_directory, exist_ok=True)

        # Save model weights
        weights_path = os.path.join(save_directory, WEIGHTS_NAME)
        model_to_save = unwrap_model(self)
        state_dict = model_to_save.state_dict() if state_dict is None else state_dict
        save_function(state_dict, weights_path)

        # Save configuration
        if save_config and hasattr(self, "config"):
            self.config.save_pretrained(save_directory)
            
 class MyModel(
        nn.Module,
        CustomPyTorchModelHubMixin,
    ):
    def __init__(self, hidden_size: int = 512, vocab_size: int = 30000, output_size: int = 4):
        super().__init__()
        self.param = nn.Parameter(torch.rand(hidden_size, vocab_size))
        self.linear = nn.Linear(output_size, vocab_size)