Skip to content

Bfloat zero #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mali-git wants to merge 16 commits into
base: master
Choose a base branch
from
Open

Bfloat zero #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
b0eb1c5
Allreduce bfloat in fp32 in zero
Sep 7, 2022
ecb85a5
Tied grad reduce for bfloat
Sep 7, 2022
55bf9f8
Correct op_builder import
rrutmann Sep 20, 2022
f58fc1b
Correct import from git_version_info
rrutmann Sep 20, 2022
e794448
Test communication type
mali-git Oct 24, 2022
2dfdce5
Merge branch 'bfloat_zero' of https://github.com/OpenGPTX/DeepSpeed i…
rrutmann Oct 24, 2022
1466e72
Test parameters' type & communication type
mali-git Oct 25, 2022
390562e
Merge branch 'bfloat_zero' of https://github.com/OpenGPTX/DeepSpeed i…
rrutmann Oct 25, 2022
5ee97ef
Test Zero stage 0 and 1
mali-git Oct 25, 2022
86d5b42
Test _exec_reduce_tied_grads
rrutmann Oct 26, 2022
c6ba9ae
Add TiedLinearStackPipe
rrutmann Oct 26, 2022
2a39442
Resolved merge conflicts
rrutmann Oct 26, 2022
12c4315
Initial draft of test__exec_backward_pass
rrutmann Nov 8, 2022
d5d8ea1
Adapt assertion
mali-git Nov 8, 2022
2d343ff
Add comments
mali-git Nov 29, 2022
256ca0b
Add comment
mali-git Nov 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deepspeed/env_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import deepspeed
import subprocess
import argparse
from .ops.op_builder import ALL_OPS
from op_builder import ALL_OPS
from .git_version_info import installed_ops, torch_info

GREEN = '\033[92m'
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/adagrad/cpu_adagrad.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
'''

import torch
from ..op_builder import CPUAdagradBuilder
from op_builder import CPUAdagradBuilder
from deepspeed.utils.logging import should_log_le


Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/adam/cpu_adam.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import torch
from cpuinfo import get_cpu_info
from ..op_builder import CPUAdamBuilder
from op_builder import CPUAdamBuilder
from deepspeed.utils import logger
from deepspeed.utils.logging import should_log_le

Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/adam/fused_adam.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from .multi_tensor_apply import MultiTensorApply

multi_tensor_applier = MultiTensorApply(2048 * 32)
from ..op_builder import FusedAdamBuilder
from op_builder import FusedAdamBuilder


class FusedAdam(torch.optim.Optimizer):
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/aio/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
Licensed under the MIT license.
'''

from ..op_builder import AsyncIOBuilder
from op_builder import AsyncIOBuilder
2 changes: 1 addition & 1 deletion deepspeed/ops/lamb/fused_lamb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
'''
import types
import torch
from ..op_builder import FusedLambBuilder
from op_builder import FusedLambBuilder


class FusedLamb(torch.optim.Optimizer):
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/quantizer/quantizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
'''
import torch

from ..op_builder import QuantizerBuilder
from op_builder import QuantizerBuilder

# Cuda modules will be imported if needed
quantizer_cuda_module = None
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/transformer/inference/moe_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import math
import torch
from torch.autograd import Function
from ... import op_builder
import op_builder
#from ...inference.engine import inference_cuda_module, specialized_mode
# Cuda modules will be imported if needed
inference_cuda_module = None
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/transformer/inference/transformer_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import math
import torch
from torch.autograd import Function
from ... import op_builder
import op_builder
import torch.nn as nn
from deepspeed import comm as dist
from deepspeed.utils.logging import log_dist
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/ops/transformer/transformer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from torch import nn
from torch.autograd import Function

from ..op_builder import TransformerBuilder, StochasticTransformerBuilder
from op_builder import TransformerBuilder, StochasticTransformerBuilder

# Cuda modules will be imported if needed
transformer_cuda_module = None
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/runtime/bf16_optimizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import os
from deepspeed import comm as dist
from deepspeed.runtime.constants import PIPE_REPLICATED
from deepspeed.ops.op_builder import UtilsBuilder
from op_builder import UtilsBuilder
from deepspeed.runtime import ZeROOptimizer
from packaging import version as pkg_version

Expand Down
Loading