%load_ext autoreload
%autoreload 2

import os
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

Imports

Utils

generate_fake_data[source]

generate_fake_data(output_format='list_tuple', label_format='string')

create_fake_label_encoder[source]

create_fake_label_encoder(params, problem:str, mode:str, label_format='string')

Local problems

get_fake_contrastive_learning_fn[source]

get_fake_contrastive_learning_fn(file_path)

get_weibo_fake_cls_fn[source]

get_weibo_fake_cls_fn(file_path)

get_weibo_fake_ner_fn[source]

get_weibo_fake_ner_fn(file_path)

get_weibo_pretrain_fn[source]

get_weibo_pretrain_fn(file_path)

get_weibo_fake_multi_cls_fn[source]

get_weibo_fake_multi_cls_fn(file_path)

get_weibo_masklm[source]

get_weibo_masklm(file_path)

get_weibo_premask_mlm[source]

get_weibo_premask_mlm(file_path)

get_weibo_fake_regression[source]

get_weibo_fake_regression(file_path)

get_weibo_fake_vector_fit[source]

get_weibo_fake_vector_fit(file_path)

generate_fake_data(output_format='dict_tuple', label_format='seq_tag_string')

({'text': ['this is a test',
   'this is a test',
   'this is a test',
   'this is a test',
   'this is a test',
   'this is a test',
   'this is a test',
   'this is a test',
   'this is a test',
   'this is a test'],
  'array': [array([0.49095954, 0.20541639, 0.49262084, 0.87451381, 0.02458258,
          0.65257202, 0.98258836, 0.58114124, 0.58505677, 0.15806585]),
   array([0.44947885, 0.03171102, 0.62730992, 0.7299056 , 0.04007493,
          0.65133139, 0.60520342, 0.15861278, 0.29668022, 0.33993872]),
   array([0.2731397 , 0.01539462, 0.53834935, 0.32880503, 0.53450392,
          0.76220641, 0.44569313, 0.78168369, 0.54579642, 0.71716377]),
   array([0.12633738, 0.75136335, 0.32964124, 0.64901069, 0.09048094,
          0.96317688, 0.47199601, 0.45125637, 0.53099414, 0.51856527]),
   array([0.75426461, 0.76001104, 0.22953306, 0.9394916 , 0.37764847,
          0.50555047, 0.93132036, 0.47237353, 0.48608123, 0.0177382 ]),
   array([0.4976838 , 0.06866748, 0.37725227, 0.12630836, 0.23961906,
          0.21206261, 0.60618255, 0.49934502, 0.28125586, 0.98571187]),
   array([0.48514133, 0.24224251, 0.88660613, 0.55056173, 0.4166745 ,
          0.09398377, 0.46331419, 0.35790425, 0.53340011, 0.50248541]),
   array([0.37552715, 0.22720832, 0.5789555 , 0.60622229, 0.29577143,
          0.52823125, 0.74576145, 0.22046372, 0.67712443, 0.06888701]),
   array([0.07969023, 0.81459337, 0.011728  , 0.23679496, 0.18646781,
          0.22561861, 0.89018834, 0.234746  , 0.36101215, 0.19218532]),
   array([0.19514571, 0.74129395, 0.10188232, 0.13237138, 0.5208849 ,
          0.24629547, 0.86091529, 0.09563499, 0.59220658, 0.58684688])],
  'cate': ['fake_cate',
   'fake_cate',
   'fake_cate',
   'fake_cate',
   'fake_cate',
   'fake_cate',
   'fake_cate',
   'fake_cate',
   'fake_cate',
   'fake_cate']},
 [['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd'],
  ['a', 'b', 'c', 'd']])

Pyspark Problems

preprocessing_fn..wrapper[source]

preprocessing_fn..wrapper(params, mode, get_data_num=False, write_tfrecord=True)

preprocessing_fn..wrapper[source]

preprocessing_fn..wrapper(params, mode, get_data_num=False, write_tfrecord=True)

preprocessing_fn..wrapper[source]

preprocessing_fn..wrapper(params, mode, get_data_num=False, write_tfrecord=True)