# import tensorflow as tf
# import numpy as np
# from m3tl.test_base import TestBase
# from m3tl.input_fn import train_eval_input_fn
# from m3tl.test_base import test_top_layer
# test_base = TestBase()
# params = test_base.params

# params.assign_problem('weibo_fake_ner&weibo_fake_cls|weibo_fake_multi_cls|weibo_masklm|weibo_pretrain',base_dir=test_base.tmpckptdir)

# hidden_dim = params.bert_config.hidden_size

# train_dataset = train_eval_input_fn(params=params)
# one_batch = next(train_dataset.as_numpy_iterator())

Imports and utils

Top Layer

class PreTrain[source]

PreTrain(*args, **kwargs) :: Model

Model groups layers into an object with training and inference features.

Arguments: inputs: The input(s) of the model: a keras.Input object or list of keras.Input objects. outputs: The output(s) of the model. See Functional API example below. name: String, the name of the model.

There are two ways to instantiate a Model:

1 - With the "Functional API", where you start from Input, you chain layer calls to specify the model's forward pass, and finally you create your model from inputs and outputs:

import tensorflow as tf

inputs = tf.keras.Input(shape=(3,))
x = tf.keras.layers.Dense(4, activation=tf.nn.relu)(inputs)
outputs = tf.keras.layers.Dense(5, activation=tf.nn.softmax)(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)

2 - By subclassing the Model class: in that case, you should define your layers in __init__ and you should implement the model's forward pass in call.

import tensorflow as tf

class MyModel(tf.keras.Model):

  def __init__(self):
    super(MyModel, self).__init__()
    self.dense1 = tf.keras.layers.Dense(4, activation=tf.nn.relu)
    self.dense2 = tf.keras.layers.Dense(5, activation=tf.nn.softmax)

  def call(self, inputs):
    x = self.dense1(inputs)
    return self.dense2(x)

model = MyModel()

If you subclass Model, you can optionally have a training argument (boolean) in call, which you can use to specify a different behavior in training and inference:

import tensorflow as tf

class MyModel(tf.keras.Model):

  def __init__(self):
    super(MyModel, self).__init__()
    self.dense1 = tf.keras.layers.Dense(4, activation=tf.nn.relu)
    self.dense2 = tf.keras.layers.Dense(5, activation=tf.nn.softmax)
    self.dropout = tf.keras.layers.Dropout(0.5)

  def call(self, inputs, training=False):
    x = self.dense1(inputs)
    if training:
      x = self.dropout(x, training=training)
    return self.dense2(x)

model = MyModel()

Once the model is created, you can config the model with losses and metrics with model.compile(), train the model with model.fit(), or use the model to do prediction with model.predict().

Get or make label encoder function

pretrain_get_or_make_label_encoder_fn[source]

pretrain_get_or_make_label_encoder_fn(params:BaseParams, problem:str, mode:str, label_list:List[str], *args, **kwargs)

Label handing function

pretrain_label_handling_fn[source]

pretrain_label_handling_fn(target, label_encoder=None, tokenizer=None, decoding_length=None, *args, **kwargs)