Index

A

add_cli_args() (sotastream.pipelines.base.DocumentPipeline class method) (sotastream.pipelines.base.Pipeline class method) (sotastream.pipelines.mtdata_pipeline.MTDataPipeline class method) (sotastream.pipelines.multistream_pipeline.MultiStreamPipeline class method)

add_global_args() (in module sotastream.cli) adjustSeed() (in module sotastream.cli) Append() (in module sotastream.augmentors.augmentors) append() (sotastream.data.Line method)

B

BitextFilter() (in module sotastream.filters.filters)

BUFFER_SIZE (sotastream.Defaults attribute)

C

canBeLowercased() (in module sotastream.augmentors.augmentors) canBeUppercased() (in module sotastream.augmentors.augmentors) compute_md5() (in module sotastream.utils.split) computePhraseSpans() (sotastream.utils.phrases.PhraseSpanExtractor method)

Copy() (in module sotastream.augmentors.augmentors) CopySource() (in module sotastream.augmentors.augmentors) create() (sotastream.pipelines.base.Pipeline static method) create_data_stream() (sotastream.pipelines.base.Pipeline method)

D

DataSource() (in module sotastream.augmentors.augmentors) DefaultPipeline (class in sotastream.pipelines.default) Defaults (class in sotastream) description (sotastream.pipelines.example_pipeline.ExamplePipeline attribute)

DOC_PROB (sotastream.Defaults attribute) DOC_PROB_PARALLEL (sotastream.Defaults attribute) DOC_SEPARATOR (sotastream.Defaults attribute) DocumentPipeline (class in sotastream.pipelines.base)

E

enumerate_files() (in module sotastream.augmentors.augmentors)

ExamplePipeline (class in sotastream.pipelines.example_pipeline) extract() (sotastream.utils.phrases.PhraseSpanExtractor method)

F

fields (sotastream.data.Line attribute)

G

get_data_sources_default_weights() (sotastream.pipelines.base.Pipeline class method) (sotastream.pipelines.default.DefaultPipeline class method) (sotastream.pipelines.example_pipeline.ExamplePipeline class method) (sotastream.pipelines.mtdata_pipeline.MTDataPipeline class method) (sotastream.pipelines.multistream_pipeline.MultiStreamPipeline class method)

get_data_sources_for_argparse() (sotastream.pipelines.base.Pipeline class method) (sotastream.pipelines.default.DefaultPipeline class method) (sotastream.pipelines.example_pipeline.ExamplePipeline class method) (sotastream.pipelines.mtdata_pipeline.MTDataPipeline class method) (sotastream.pipelines.multistream_pipeline.MultiStreamPipeline class method) getPhraseSpans() (sotastream.utils.phrases.PhraseSpanExtractor method)

I

Identity() (in module sotastream.augmentors.augmentors)

J

join() (sotastream.data.Line static method)

JustSourceTarget() (in module sotastream.augmentors.augmentors)

L

Line (class in sotastream.data)

LowerCase() (in module sotastream.pipelines.example_pipeline)

M

main() (in module sotastream.cli) MatchFilter() (in module sotastream.filters.filters) MAX_TOKENS (sotastream.Defaults attribute) maybe_split_files() (in module sotastream.cli) Mixer (class in sotastream.augmentors.augmentors) module sotastream sotastream.augmentors sotastream.augmentors.augmentors sotastream.cli sotastream.data sotastream.filters sotastream.filters.filters sotastream.pipelines sotastream.pipelines.base sotastream.pipelines.default sotastream.pipelines.example_pipeline sotastream.pipelines.mtdata_pipeline sotastream.pipelines.multistream_pipeline sotastream.utils sotastream.utils.phrases sotastream.utils.split

MTDataPipeline (class in sotastream.pipelines.mtdata_pipeline) MTDataSource() (in module sotastream.pipelines.mtdata_pipeline) Multiply() (in module sotastream.augmentors.augmentors) MultiStreamPipeline (class in sotastream.pipelines.multistream_pipeline)

N

NUM_PROCESSES (sotastream.Defaults attribute)

P

PhraseSpanExtractor (class in sotastream.utils.phrases)

Pipeline (class in sotastream.pipelines.base)

Q

QUEUE_BUFFER_SIZE (sotastream.Defaults attribute)

QUIET (sotastream.Defaults attribute)

R

ReadAndAugment() (in module sotastream.pipelines.example_pipeline)

RegexFilter() (in module sotastream.filters.filters) run_pipeline_process() (in module sotastream.cli)

S

SAMPLE_FILE (sotastream.Defaults attribute) SAMPLE_LENGTH (sotastream.Defaults attribute) samplePhraseSpans() (sotastream.utils.phrases.PhraseSpanExtractor method) SEED (sotastream.Defaults attribute) SEPARATOR (sotastream.Defaults attribute) SHUFFLE (sotastream.Defaults attribute) SkipBlanks() (in module sotastream.filters.filters) smart_open() (in module sotastream.utils.split) sotastream module sotastream.augmentors module sotastream.augmentors.augmentors module sotastream.cli module sotastream.data module sotastream.filters module sotastream.filters.filters module sotastream.pipelines module

sotastream.pipelines.base module sotastream.pipelines.default module sotastream.pipelines.example_pipeline module sotastream.pipelines.mtdata_pipeline module sotastream.pipelines.multistream_pipeline module sotastream.utils module sotastream.utils.phrases module sotastream.utils.split module split_file_into_chunks() (in module sotastream.utils.split) split_native() (in module sotastream.utils.split) split_subshell() (in module sotastream.utils.split) SPM_MODEL (sotastream.Defaults attribute) SPMDecoder() (in module sotastream.augmentors.augmentors) SPMEncoder() (in module sotastream.augmentors.augmentors)

T

TagData() (in module sotastream.pipelines.example_pipeline) Tagger() (in module sotastream.augmentors.augmentors) TitleCase() (in module sotastream.pipelines.example_pipeline)

ToLower() (in module sotastream.augmentors.augmentors) ToTitle() (in module sotastream.augmentors.augmentors) ToUpper() (in module sotastream.augmentors.augmentors)

U

UTF8File() (in module sotastream.augmentors.augmentors)