pr review

Author: Francisco Santos

.gitignore

@@ -373,4 +373,6 @@ DerivedData/
 
 # User created
 VERSION
-version.py
+version.py
+local_test_*.py
+local_test_*.ipynb

src/ydata_synthetic/postprocessing/regular/inverse_preprocesser.py

@@ -1,45 +1,46 @@
 # Inverts all preprocessing pipelines provided in the preprocessing examples
 from typing import Union
 
-import pandas as pd
+from pandas import DataFrame, concat
 
 from sklearn.pipeline import Pipeline
 from sklearn.compose import ColumnTransformer
-from sklearn.preprocessing import PowerTransformer, OneHotEncoder, StandardScaler
+from sklearn.preprocessing import PowerTransformer, OneHotEncoder, StandardScaler, MinMaxScaler
 
 
-def inverse_transform(data: pd.DataFrame, processor: Union[Pipeline, ColumnTransformer, PowerTransformer, OneHotEncoder, StandardScaler]) -> pd.DataFrame:
+def inverse_transform(data: DataFrame, processor: Union[Pipeline, ColumnTransformer, PowerTransformer,
+                                                           OneHotEncoder, StandardScaler, MinMaxScaler]) -> DataFrame:
     """Inverts data transformations taking place in a standard sklearn processor.
     Supported processes are sklearn pipelines, column transformers or base estimators like standard scalers.
 
     Args:
-        data (pd.DataFrame): The data object that needs inversion of preprocessing
+        data (DataFrame): The data object that needs inversion of preprocessing
         processor (Union[Pipeline, ColumnTransformer, BaseEstimator]): The processor applied on the original data
 
     Returns:
-        inv_data (pd.DataFrame): The data object after inverting preprocessing"""
+        inv_data (DataFrame): The data object after inverting preprocessing"""
     inv_data = data.copy()
-    if isinstance(processor, (PowerTransformer, OneHotEncoder, StandardScaler, Pipeline)):
-        inv_data = pd.DataFrame(processor.inverse_transform(data), columns=processor.feature_names_in_)
+    if isinstance(processor, (PowerTransformer, OneHotEncoder, StandardScaler, MinMaxScaler, Pipeline)):
+        inv_data = DataFrame(processor.inverse_transform(data), columns=processor.feature_names_in_ if hasattr(processor, "feature_names_in") else None)
     elif isinstance(processor, ColumnTransformer):
         output_indices = processor.output_indices_
-        assert isinstance(data, pd.DataFrame), "The data to be inverted from a ColumnTransformer has to be a Pandas DataFrame."
+        assert isinstance(data, DataFrame), "The data to be inverted from a ColumnTransformer has to be a Pandas DataFrame."
         for t_name, t, t_cols in processor.transformers_[::-1]:
             slice_ = output_indices[t_name]
             t_indices = list(range(slice_.start, slice_.stop, 1 if slice_.step is None else slice_.step))
             if t == 'drop':
                 continue
             elif t == 'passthrough':
-                inv_cols = pd.DataFrame(data.iloc[:,t_indices].values, columns = t_cols, index = data.index)
+                inv_cols = DataFrame(data.iloc[:,t_indices].values, columns = t_cols, index = data.index)
                 inv_col_names = inv_cols.columns
             else:
-                inv_cols = pd.DataFrame(t.inverse_transform(data.iloc[:,t_indices].values), columns = t_cols, index = data.index)
+                inv_cols = DataFrame(t.inverse_transform(data.iloc[:,t_indices].values), columns = t_cols, index = data.index)
                 inv_col_names = inv_cols.columns
             if set(inv_col_names).issubset(set(inv_data.columns)):
                 inv_data[inv_col_names] = inv_cols[inv_col_names]
             else:
-                inv_data = pd.concat([inv_data, inv_cols], axis=1)
+                inv_data = concat([inv_data, inv_cols], axis=1)
     else:
         print('The provided data processor is not supported and cannot be inverted with this method.')
         return None
-    return inv_data[processor.feature_names_in_]
+    return inv_data[processor.feature_names_in_] if hasattr(processor, "feature_names_in") else inv_data

src/ydata_synthetic/postprocessing/timeseries/inverse_preprocesser.py

@@ -0,0 +1,17 @@
+from typing import Union, List
+
+from ydata_synthetic.postprocessing.regular import inverse_preprocesser
+
+from sklearn.pipeline import Pipeline
+from sklearn.compose import ColumnTransformer
+from sklearn.preprocessing import PowerTransformer, OneHotEncoder, StandardScaler, MinMaxScaler
+
+from pandas import DataFrame
+
+def inverse_transform(data: List, processor: Union[Pipeline, ColumnTransformer, PowerTransformer, OneHotEncoder,
+                                                   StandardScaler, MinMaxScaler]):
+    if isinstance(data, list):
+        data = DataFrame(data)
+        return inverse_preprocesser.inverse_transform(data, processor).tolist()
+    else:
+        return inverse_preprocesser.inverse_transform(data, processor)

src/ydata_synthetic/preprocessing/timeseries/__init__.py

@@ -1,7 +1,5 @@
 from ydata_synthetic.preprocessing.timeseries.stock import transformations as processed_stock
-from ydata_synthetic.preprocessing.timeseries.stock_univariate import transformations as processed_stock_univariate
 
 __all__ = [
     "processed_stock",
-    "processed_stock_univariate"
 ]

src/ydata_synthetic/preprocessing/timeseries/stock.py

@@ -2,12 +2,25 @@
     Get the stock data from Yahoo finance data
     Data from the period 01 January 2017 - 24 January 2021
 """
+from typing import Union, List
+
 import pandas as pd
 
 from ydata_synthetic.preprocessing.timeseries.utils import real_data_loading
 
-def transformations(path, seq_len: int):
-    stock_df = pd.read_csv(path)
+def transformations(path, seq_len: int, cols: Union[str, List] = None):
+    """Apply min max scaling and roll windows of a temporal dataset.
+
+    Args:
+        path(str): path to a csv temporal dataframe
+        seq_len(int): length of the rolled sequences
+        cols (Union[str, List]): Column or list of columns to be used"""
+    if isinstance(cols, str):
+        cols = [cols]
+    if isinstance(cols, list):
+        stock_df = pd.read_csv(path)[cols]
+    else:
+        stock_df = pd.read_csv(path)
     try:
         stock_df = stock_df.set_index('Date').sort_index()
     except:

src/ydata_synthetic/preprocessing/timeseries/stock_univariate.py

@@ -25,6 +25,7 @@ class TSCWGAN(BaseModel):
     def __init__(self, model_parameters, gradient_penalty_weight=10):
         """Create a base TSCWGAN."""
         self.gradient_penalty_weight = gradient_penalty_weight
+        self.cond_dim = model_parameters.condition
         super().__init__(model_parameters)
 
     def define_gan(self):
@@ -170,91 +171,78 @@ def __init__(self, batch_size):
         self.batch_size = batch_size
 
     def build_model(self, input_shape, dim, data_dim):
-        # Define blocks
-        input_to_latent = Sequential(layers=[
+        # Define input - Expected input shape is (batch_size, seq_len, noise_dim). noise_dim = Z + cond
+        noise_input = Input(shape = input_shape, batch_size = self.batch_size)
+
+        # Compose model
+        proc_input = Sequential(layers=[
             Conv1D(filters=dim, kernel_size=1, input_shape = input_shape),
             LeakyReLU(),
             Conv1D(dim, kernel_size=5, dilation_rate=2, padding="same"),
             LeakyReLU()
-        ], name='input_to_latent')
+        ], name='input_to_latent')(noise_input)
+
         block_cnn = Sequential(layers=[
             Conv1D(filters=dim, kernel_size=3, dilation_rate=2, padding="same"),
             LeakyReLU()
         ], name='block_cnn')
-        block_shift = Sequential(layers=[
+        for i in range(3):
+            if i == 0:
+                cnn_block_i = proc_input
+                cnn_block_o = block_cnn(proc_input)
+            else:
+                cnn_block_o = block_cnn(cnn_block_i)
+            cnn_block_i = Add()([cnn_block_i, cnn_block_o])
+
+        shift = Sequential(layers=[
             Conv1D(filters=10, kernel_size=3, dilation_rate=2, padding="same"),
             LeakyReLU(),
             Flatten(),
             Dense(dim*2),
             LeakyReLU()
-        ], name='block_shift')
+        ], name='block_shift')(cnn_block_i)
+
         block = Sequential(layers=[
             Dense(dim*2),
             LeakyReLU()
         ], name='block')
-        latent_to_output = Sequential([
-            Dense(data_dim)
-        ], name='latent_to_ouput')
+        for i in range(3):
+            if i == 0:
+                block_i = shift
+                block_o = block(shift)
+            else:
+                block_o = block(block_i)
+            block_i = Add()([block_i, block_o])
 
-        # Define input - Expected input shape is (batch_size, seq_len, noise_dim). noise_dim = Z + cond
-        noise_input = Input(shape = input_shape, batch_size = self.batch_size)
-
-        # Compose model
-        x = input_to_latent(noise_input)
-        x_block = block_cnn(x)
-        x = Add()([x_block, x])
-        x_block = block_cnn(x)
-        x = Add()([x_block, x])
-        x_block = block_cnn(x)
-        x = Add()([x_block, x])
-        x = block_shift(x)
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x = latent_to_output(x)
-        # Output - Expected shape is (batch_size, seq_len, data_dim). data_dim does not include conditions
-        return Model(inputs=noise_input, outputs=x, name='SkipConnectionGenerator')
+        output = Dense(data_dim, name='latent_to_ouput')(block_i)
+        return Model(inputs = noise_input, outputs = output, name='SkipConnectionGenerator')
 
 class Critic(Model):
     """Conditional Wasserstein Critic with skip connections."""
     def __init__(self, batch_size):
         self.batch_size = batch_size
 
     def build_model(self, input_shape, dim):
-        # Define blocks
-        ts_to_latent = Sequential(layers=[
+        # Define input - Expected input shape is X + condition
+        record_input = Input(shape = input_shape, batch_size = self.batch_size)
+
+        # Compose model
+        proc_record = Sequential(layers=[
             Dense(dim*2,),
             LeakyReLU()
-        ], name='ts_to_latent')
+        ], name='ts_to_latent')(record_input)
+
         block = Sequential(layers=[
             Dense(dim*2),
             LeakyReLU()
         ], name='block')
-        latent_to_score = Sequential(layers=[
-            Dense(1)
-        ], name='latent_to_score')
-
-        # Define input - Expected input shape is X + condition
-        record_input = Input(shape = input_shape, batch_size = self.batch_size)
-
-        # Compose model
-        x = ts_to_latent(record_input)
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x_block = block(x)
-        x = Add()([x_block, x])
-        x = latent_to_score(x)
-        return Model(inputs=record_input, outputs=x, name='SkipConnectionCritic')
+        for i in range(7):
+            if i == 0:
+                block_i = proc_record
+                block_o = block(proc_record)
+            else:
+                block_o = block(block_i)
+            block_i = Add()([block_i, block_o])
+
+        output = Dense(1, name = 'latent_to_score')(block_i)
+        return Model(inputs=record_input, outputs=output, name='SkipConnectionCritic')