1 # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """Utils for testing linear estimators.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 import math 22 import os 23 import shutil 24 import tempfile 25 26 import numpy as np 27 import six 28 29 from tensorflow.core.example import example_pb2 30 from tensorflow.core.example import feature_pb2 31 from tensorflow.python.client import session as tf_session 32 from tensorflow.python.estimator import estimator 33 from tensorflow.python.estimator import run_config 34 from tensorflow.python.estimator import warm_starting_util 35 from tensorflow.python.estimator.canned import linear 36 from tensorflow.python.estimator.canned import metric_keys 37 from tensorflow.python.estimator.export import export 38 from tensorflow.python.estimator.inputs import numpy_io 39 from tensorflow.python.estimator.inputs import pandas_io 40 from tensorflow.python.feature_column import feature_column as feature_column_lib 41 from tensorflow.python.framework import dtypes 42 from tensorflow.python.framework import ops 43 from tensorflow.python.framework import sparse_tensor 44 from tensorflow.python.ops import check_ops 45 from tensorflow.python.ops import control_flow_ops 46 from tensorflow.python.ops import data_flow_ops 47 from tensorflow.python.ops import init_ops 48 from tensorflow.python.ops import math_ops 49 from tensorflow.python.ops import parsing_ops 50 from tensorflow.python.ops import partitioned_variables 51 from tensorflow.python.ops import state_ops 52 from tensorflow.python.ops import variable_scope 53 from tensorflow.python.ops import variables as variables_lib 54 from tensorflow.python.platform import gfile 55 from tensorflow.python.platform import test 56 from tensorflow.python.summary.writer import writer_cache 57 from tensorflow.python.training import checkpoint_utils 58 from tensorflow.python.training import gradient_descent 59 from tensorflow.python.training import input as input_lib 60 from tensorflow.python.training import optimizer as optimizer_lib 61 from tensorflow.python.training import queue_runner 62 from tensorflow.python.training import saver 63 from tensorflow.python.training import session_run_hook 64 65 try: 66 # pylint: disable=g-import-not-at-top 67 import pandas as pd 68 HAS_PANDAS = True 69 except IOError: 70 # Pandas writes a temporary file during import. If it fails, don't use pandas. 71 HAS_PANDAS = False 72 except ImportError: 73 HAS_PANDAS = False 74 75 # pylint rules which are disabled by default for test files. 76 # pylint: disable=invalid-name,protected-access,missing-docstring 77 78 # Names of variables created by model. 79 AGE_WEIGHT_NAME = 'linear/linear_model/age/weights' 80 HEIGHT_WEIGHT_NAME = 'linear/linear_model/height/weights' 81 OCCUPATION_WEIGHT_NAME = 'linear/linear_model/occupation/weights' 82 BIAS_NAME = 'linear/linear_model/bias_weights' 83 LANGUAGE_WEIGHT_NAME = 'linear/linear_model/language/weights' 84 85 86 def assert_close(expected, actual, rtol=1e-04, name='assert_close'): 87 with ops.name_scope(name, 'assert_close', (expected, actual, rtol)) as scope: 88 expected = ops.convert_to_tensor(expected, name='expected') 89 actual = ops.convert_to_tensor(actual, name='actual') 90 rdiff = math_ops.abs(expected - actual, 'diff') / math_ops.abs(expected) 91 rtol = ops.convert_to_tensor(rtol, name='rtol') 92 return check_ops.assert_less( 93 rdiff, 94 rtol, 95 data=('Condition expected =~ actual did not hold element-wise:' 96 'expected = ', expected, 'actual = ', actual, 'rdiff = ', rdiff, 97 'rtol = ', rtol,), 98 name=scope) 99 100 101 def save_variables_to_ckpt(model_dir): 102 init_all_op = [variables_lib.global_variables_initializer()] 103 with tf_session.Session() as sess: 104 sess.run(init_all_op) 105 saver.Saver().save(sess, os.path.join(model_dir, 'model.ckpt')) 106 107 108 def queue_parsed_features(feature_map): 109 tensors_to_enqueue = [] 110 keys = [] 111 for key, tensor in six.iteritems(feature_map): 112 keys.append(key) 113 tensors_to_enqueue.append(tensor) 114 queue_dtypes = [x.dtype for x in tensors_to_enqueue] 115 input_queue = data_flow_ops.FIFOQueue(capacity=100, dtypes=queue_dtypes) 116 queue_runner.add_queue_runner( 117 queue_runner.QueueRunner(input_queue, 118 [input_queue.enqueue(tensors_to_enqueue)])) 119 dequeued_tensors = input_queue.dequeue() 120 return {keys[i]: dequeued_tensors[i] for i in range(len(dequeued_tensors))} 121 122 123 def sorted_key_dict(unsorted_dict): 124 return {k: unsorted_dict[k] for k in sorted(unsorted_dict)} 125 126 127 def sigmoid(x): 128 return 1 / (1 + np.exp(-1.0 * x)) 129 130 131 class CheckPartitionerVarHook(session_run_hook.SessionRunHook): 132 """A `SessionRunHook` to check a partitioned variable.""" 133 134 def __init__(self, test_case, var_name, var_dim, partitions): 135 self._test_case = test_case 136 self._var_name = var_name 137 self._var_dim = var_dim 138 self._partitions = partitions 139 140 def begin(self): 141 with variable_scope.variable_scope( 142 variable_scope.get_variable_scope()) as scope: 143 scope.reuse_variables() 144 partitioned_weight = variable_scope.get_variable( 145 self._var_name, shape=(self._var_dim, 1)) 146 self._test_case.assertTrue( 147 isinstance(partitioned_weight, variables_lib.PartitionedVariable)) 148 for part in partitioned_weight: 149 self._test_case.assertEqual(self._var_dim // self._partitions, 150 part.get_shape()[0]) 151 152 153 class BaseLinearRegressorPartitionerTest(object): 154 155 def __init__(self, linear_regressor_fn): 156 self._linear_regressor_fn = linear_regressor_fn 157 158 def setUp(self): 159 self._model_dir = tempfile.mkdtemp() 160 161 def tearDown(self): 162 if self._model_dir: 163 writer_cache.FileWriterCache.clear() 164 shutil.rmtree(self._model_dir) 165 166 def testPartitioner(self): 167 x_dim = 64 168 partitions = 4 169 170 def _partitioner(shape, dtype): 171 del dtype # unused; required by Fn signature. 172 # Only partition the embedding tensor. 173 return [partitions, 1] if shape[0] == x_dim else [1] 174 175 regressor = self._linear_regressor_fn( 176 feature_columns=(feature_column_lib.categorical_column_with_hash_bucket( 177 'language', hash_bucket_size=x_dim),), 178 partitioner=_partitioner, 179 model_dir=self._model_dir) 180 181 def _input_fn(): 182 return { 183 'language': 184 sparse_tensor.SparseTensor( 185 values=['english', 'spanish'], 186 indices=[[0, 0], [0, 1]], 187 dense_shape=[1, 2]) 188 }, [[10.]] 189 190 hook = CheckPartitionerVarHook(self, LANGUAGE_WEIGHT_NAME, x_dim, 191 partitions) 192 regressor.train(input_fn=_input_fn, steps=1, hooks=[hook]) 193 194 def testDefaultPartitionerWithMultiplePsReplicas(self): 195 partitions = 2 196 # This results in weights larger than the default partition size of 64M, 197 # so partitioned weights are created (each weight uses 4 bytes). 198 x_dim = 32 << 20 199 200 class FakeRunConfig(run_config.RunConfig): 201 202 @property 203 def num_ps_replicas(self): 204 return partitions 205 206 # Mock the device setter as ps is not available on test machines. 207 with test.mock.patch.object( 208 estimator, 209 '_get_replica_device_setter', 210 return_value=lambda _: '/cpu:0'): 211 linear_regressor = self._linear_regressor_fn( 212 feature_columns=( 213 feature_column_lib.categorical_column_with_hash_bucket( 214 'language', hash_bucket_size=x_dim),), 215 config=FakeRunConfig(), 216 model_dir=self._model_dir) 217 218 def _input_fn(): 219 return { 220 'language': 221 sparse_tensor.SparseTensor( 222 values=['english', 'spanish'], 223 indices=[[0, 0], [0, 1]], 224 dense_shape=[1, 2]) 225 }, [[10.]] 226 227 hook = CheckPartitionerVarHook(self, LANGUAGE_WEIGHT_NAME, x_dim, 228 partitions) 229 linear_regressor.train(input_fn=_input_fn, steps=1, hooks=[hook]) 230 231 232 # TODO(b/36813849): Add tests with dynamic shape inputs using placeholders. 233 class BaseLinearRegressorEvaluationTest(object): 234 235 def __init__(self, linear_regressor_fn): 236 self._linear_regressor_fn = linear_regressor_fn 237 238 def setUp(self): 239 self._model_dir = tempfile.mkdtemp() 240 241 def tearDown(self): 242 if self._model_dir: 243 writer_cache.FileWriterCache.clear() 244 shutil.rmtree(self._model_dir) 245 246 def test_evaluation_for_simple_data(self): 247 with ops.Graph().as_default(): 248 variables_lib.Variable([[11.0]], name=AGE_WEIGHT_NAME) 249 variables_lib.Variable([2.0], name=BIAS_NAME) 250 variables_lib.Variable( 251 100, name=ops.GraphKeys.GLOBAL_STEP, dtype=dtypes.int64) 252 save_variables_to_ckpt(self._model_dir) 253 254 linear_regressor = self._linear_regressor_fn( 255 feature_columns=(feature_column_lib.numeric_column('age'),), 256 model_dir=self._model_dir) 257 eval_metrics = linear_regressor.evaluate( 258 input_fn=lambda: ({'age': ((1,),)}, ((10.,),)), steps=1) 259 260 # Logit is (1. * 11.0 + 2.0) = 13, while label is 10. Loss is 3**2 = 9. 261 self.assertDictEqual({ 262 metric_keys.MetricKeys.LOSS: 9., 263 metric_keys.MetricKeys.LOSS_MEAN: 9., 264 ops.GraphKeys.GLOBAL_STEP: 100 265 }, eval_metrics) 266 267 def test_evaluation_batch(self): 268 """Tests evaluation for batch_size==2.""" 269 with ops.Graph().as_default(): 270 variables_lib.Variable([[11.0]], name=AGE_WEIGHT_NAME) 271 variables_lib.Variable([2.0], name=BIAS_NAME) 272 variables_lib.Variable( 273 100, name=ops.GraphKeys.GLOBAL_STEP, dtype=dtypes.int64) 274 save_variables_to_ckpt(self._model_dir) 275 276 linear_regressor = self._linear_regressor_fn( 277 feature_columns=(feature_column_lib.numeric_column('age'),), 278 model_dir=self._model_dir) 279 eval_metrics = linear_regressor.evaluate( 280 input_fn=lambda: ({'age': ((1,), (1,))}, ((10.,), (10.,))), steps=1) 281 282 # Logit is (1. * 11.0 + 2.0) = 13, while label is 10. 283 # Loss per example is 3**2 = 9. 284 # Training loss is the sum over batch = 9 + 9 = 18 285 # Average loss is the average over batch = 9 286 self.assertDictEqual({ 287 metric_keys.MetricKeys.LOSS: 18., 288 metric_keys.MetricKeys.LOSS_MEAN: 9., 289 ops.GraphKeys.GLOBAL_STEP: 100 290 }, eval_metrics) 291 292 def test_evaluation_weights(self): 293 """Tests evaluation with weights.""" 294 with ops.Graph().as_default(): 295 variables_lib.Variable([[11.0]], name=AGE_WEIGHT_NAME) 296 variables_lib.Variable([2.0], name=BIAS_NAME) 297 variables_lib.Variable( 298 100, name=ops.GraphKeys.GLOBAL_STEP, dtype=dtypes.int64) 299 save_variables_to_ckpt(self._model_dir) 300 301 def _input_fn(): 302 features = {'age': ((1,), (1,)), 'weights': ((1.,), (2.,))} 303 labels = ((10.,), (10.,)) 304 return features, labels 305 306 linear_regressor = self._linear_regressor_fn( 307 feature_columns=(feature_column_lib.numeric_column('age'),), 308 weight_column='weights', 309 model_dir=self._model_dir) 310 eval_metrics = linear_regressor.evaluate(input_fn=_input_fn, steps=1) 311 312 # Logit is (1. * 11.0 + 2.0) = 13, while label is 10. 313 # Loss per example is 3**2 = 9. 314 # Training loss is the weighted sum over batch = 9 + 2*9 = 27 315 # average loss is the weighted average = 9 + 2*9 / (1 + 2) = 9 316 self.assertDictEqual({ 317 metric_keys.MetricKeys.LOSS: 27., 318 metric_keys.MetricKeys.LOSS_MEAN: 9., 319 ops.GraphKeys.GLOBAL_STEP: 100 320 }, eval_metrics) 321 322 def test_evaluation_for_multi_dimensions(self): 323 x_dim = 3 324 label_dim = 2 325 with ops.Graph().as_default(): 326 variables_lib.Variable( 327 [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], name=AGE_WEIGHT_NAME) 328 variables_lib.Variable([7.0, 8.0], name=BIAS_NAME) 329 variables_lib.Variable(100, name='global_step', dtype=dtypes.int64) 330 save_variables_to_ckpt(self._model_dir) 331 332 linear_regressor = self._linear_regressor_fn( 333 feature_columns=(feature_column_lib.numeric_column( 334 'age', shape=(x_dim,)),), 335 label_dimension=label_dim, 336 model_dir=self._model_dir) 337 input_fn = numpy_io.numpy_input_fn( 338 x={ 339 'age': np.array([[2., 4., 5.]]), 340 }, 341 y=np.array([[46., 58.]]), 342 batch_size=1, 343 num_epochs=None, 344 shuffle=False) 345 eval_metrics = linear_regressor.evaluate(input_fn=input_fn, steps=1) 346 347 self.assertItemsEqual( 348 (metric_keys.MetricKeys.LOSS, metric_keys.MetricKeys.LOSS_MEAN, 349 ops.GraphKeys.GLOBAL_STEP), eval_metrics.keys()) 350 351 # Logit is 352 # [2., 4., 5.] * [1.0, 2.0] + [7.0, 8.0] = [39, 50] + [7.0, 8.0] 353 # [3.0, 4.0] 354 # [5.0, 6.0] 355 # which is [46, 58] 356 self.assertAlmostEqual(0, eval_metrics[metric_keys.MetricKeys.LOSS]) 357 358 def test_evaluation_for_multiple_feature_columns(self): 359 with ops.Graph().as_default(): 360 variables_lib.Variable([[10.0]], name=AGE_WEIGHT_NAME) 361 variables_lib.Variable([[2.0]], name=HEIGHT_WEIGHT_NAME) 362 variables_lib.Variable([5.0], name=BIAS_NAME) 363 variables_lib.Variable( 364 100, name=ops.GraphKeys.GLOBAL_STEP, dtype=dtypes.int64) 365 save_variables_to_ckpt(self._model_dir) 366 367 batch_size = 2 368 feature_columns = [ 369 feature_column_lib.numeric_column('age'), 370 feature_column_lib.numeric_column('height') 371 ] 372 input_fn = numpy_io.numpy_input_fn( 373 x={'age': np.array([20, 40]), 374 'height': np.array([4, 8])}, 375 y=np.array([[213.], [421.]]), 376 batch_size=batch_size, 377 num_epochs=None, 378 shuffle=False) 379 380 est = self._linear_regressor_fn( 381 feature_columns=feature_columns, model_dir=self._model_dir) 382 383 eval_metrics = est.evaluate(input_fn=input_fn, steps=1) 384 self.assertItemsEqual( 385 (metric_keys.MetricKeys.LOSS, metric_keys.MetricKeys.LOSS_MEAN, 386 ops.GraphKeys.GLOBAL_STEP), eval_metrics.keys()) 387 388 # Logit is [(20. * 10.0 + 4 * 2.0 + 5.0), (40. * 10.0 + 8 * 2.0 + 5.0)] = 389 # [213.0, 421.0], while label is [213., 421.]. Loss = 0. 390 self.assertAlmostEqual(0, eval_metrics[metric_keys.MetricKeys.LOSS]) 391 392 393 class BaseLinearRegressorPredictTest(object): 394 395 def __init__(self, linear_regressor_fn): 396 self._linear_regressor_fn = linear_regressor_fn 397 398 def setUp(self): 399 self._model_dir = tempfile.mkdtemp() 400 401 def tearDown(self): 402 if self._model_dir: 403 writer_cache.FileWriterCache.clear() 404 shutil.rmtree(self._model_dir) 405 406 def test_1d(self): 407 """Tests predict when all variables are one-dimensional.""" 408 with ops.Graph().as_default(): 409 variables_lib.Variable([[10.]], name='linear/linear_model/x/weights') 410 variables_lib.Variable([.2], name=BIAS_NAME) 411 variables_lib.Variable(100, name='global_step', dtype=dtypes.int64) 412 save_variables_to_ckpt(self._model_dir) 413 414 linear_regressor = self._linear_regressor_fn( 415 feature_columns=(feature_column_lib.numeric_column('x'),), 416 model_dir=self._model_dir) 417 418 predict_input_fn = numpy_io.numpy_input_fn( 419 x={'x': np.array([[2.]])}, 420 y=None, 421 batch_size=1, 422 num_epochs=1, 423 shuffle=False) 424 predictions = linear_regressor.predict(input_fn=predict_input_fn) 425 predicted_scores = list([x['predictions'] for x in predictions]) 426 # x * weight + bias = 2. * 10. + .2 = 20.2 427 self.assertAllClose([[20.2]], predicted_scores) 428 429 def testMultiDim(self): 430 """Tests predict when all variables are multi-dimenstional.""" 431 batch_size = 2 432 label_dimension = 3 433 x_dim = 4 434 feature_columns = (feature_column_lib.numeric_column('x', shape=(x_dim,)),) 435 with ops.Graph().as_default(): 436 variables_lib.Variable( # shape=[x_dim, label_dimension] 437 [[1., 2., 3.], [2., 3., 4.], [3., 4., 5.], [4., 5., 6.]], 438 name='linear/linear_model/x/weights') 439 variables_lib.Variable( # shape=[label_dimension] 440 [.2, .4, .6], name=BIAS_NAME) 441 variables_lib.Variable(100, name='global_step', dtype=dtypes.int64) 442 save_variables_to_ckpt(self._model_dir) 443 444 linear_regressor = self._linear_regressor_fn( 445 feature_columns=feature_columns, 446 label_dimension=label_dimension, 447 model_dir=self._model_dir) 448 449 predict_input_fn = numpy_io.numpy_input_fn( 450 # x shape=[batch_size, x_dim] 451 x={'x': np.array([[1., 2., 3., 4.], [5., 6., 7., 8.]])}, 452 y=None, 453 batch_size=batch_size, 454 num_epochs=1, 455 shuffle=False) 456 predictions = linear_regressor.predict(input_fn=predict_input_fn) 457 predicted_scores = list([x['predictions'] for x in predictions]) 458 # score = x * weight + bias, shape=[batch_size, label_dimension] 459 self.assertAllClose([[30.2, 40.4, 50.6], [70.2, 96.4, 122.6]], 460 predicted_scores) 461 462 def testTwoFeatureColumns(self): 463 """Tests predict with two feature columns.""" 464 with ops.Graph().as_default(): 465 variables_lib.Variable([[10.]], name='linear/linear_model/x0/weights') 466 variables_lib.Variable([[20.]], name='linear/linear_model/x1/weights') 467 variables_lib.Variable([.2], name=BIAS_NAME) 468 variables_lib.Variable(100, name='global_step', dtype=dtypes.int64) 469 save_variables_to_ckpt(self._model_dir) 470 471 linear_regressor = self._linear_regressor_fn( 472 feature_columns=(feature_column_lib.numeric_column('x0'), 473 feature_column_lib.numeric_column('x1')), 474 model_dir=self._model_dir) 475 476 predict_input_fn = numpy_io.numpy_input_fn( 477 x={'x0': np.array([[2.]]), 478 'x1': np.array([[3.]])}, 479 y=None, 480 batch_size=1, 481 num_epochs=1, 482 shuffle=False) 483 predictions = linear_regressor.predict(input_fn=predict_input_fn) 484 predicted_scores = list([x['predictions'] for x in predictions]) 485 # x0 * weight0 + x1 * weight1 + bias = 2. * 10. + 3. * 20 + .2 = 80.2 486 self.assertAllClose([[80.2]], predicted_scores) 487 488 489 class BaseLinearRegressorIntegrationTest(object): 490 491 def __init__(self, linear_regressor_fn): 492 self._linear_regressor_fn = linear_regressor_fn 493 494 def setUp(self): 495 self._model_dir = tempfile.mkdtemp() 496 497 def tearDown(self): 498 if self._model_dir: 499 writer_cache.FileWriterCache.clear() 500 shutil.rmtree(self._model_dir) 501 502 def _test_complete_flow(self, train_input_fn, eval_input_fn, predict_input_fn, 503 input_dimension, label_dimension, prediction_length): 504 feature_columns = [ 505 feature_column_lib.numeric_column('x', shape=(input_dimension,)) 506 ] 507 est = self._linear_regressor_fn( 508 feature_columns=feature_columns, 509 label_dimension=label_dimension, 510 model_dir=self._model_dir) 511 512 # TRAIN 513 # learn y = x 514 est.train(train_input_fn, steps=200) 515 516 # EVALUTE 517 scores = est.evaluate(eval_input_fn) 518 self.assertEqual(200, scores[ops.GraphKeys.GLOBAL_STEP]) 519 self.assertIn(metric_keys.MetricKeys.LOSS, six.iterkeys(scores)) 520 521 # PREDICT 522 predictions = np.array( 523 [x['predictions'] for x in est.predict(predict_input_fn)]) 524 self.assertAllEqual((prediction_length, label_dimension), predictions.shape) 525 526 # EXPORT 527 feature_spec = feature_column_lib.make_parse_example_spec(feature_columns) 528 serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn( 529 feature_spec) 530 export_dir = est.export_savedmodel(tempfile.mkdtemp(), 531 serving_input_receiver_fn) 532 self.assertTrue(gfile.Exists(export_dir)) 533 534 def test_numpy_input_fn(self): 535 """Tests complete flow with numpy_input_fn.""" 536 label_dimension = 2 537 input_dimension = label_dimension 538 batch_size = 10 539 prediction_length = batch_size 540 data = np.linspace(0., 2., batch_size * label_dimension, dtype=np.float32) 541 data = data.reshape(batch_size, label_dimension) 542 543 train_input_fn = numpy_io.numpy_input_fn( 544 x={'x': data}, 545 y=data, 546 batch_size=batch_size, 547 num_epochs=None, 548 shuffle=True) 549 eval_input_fn = numpy_io.numpy_input_fn( 550 x={'x': data}, 551 y=data, 552 batch_size=batch_size, 553 num_epochs=1, 554 shuffle=False) 555 predict_input_fn = numpy_io.numpy_input_fn( 556 x={'x': data}, 557 y=None, 558 batch_size=batch_size, 559 num_epochs=1, 560 shuffle=False) 561 562 self._test_complete_flow( 563 train_input_fn=train_input_fn, 564 eval_input_fn=eval_input_fn, 565 predict_input_fn=predict_input_fn, 566 input_dimension=input_dimension, 567 label_dimension=label_dimension, 568 prediction_length=prediction_length) 569 570 def test_pandas_input_fn(self): 571 """Tests complete flow with pandas_input_fn.""" 572 if not HAS_PANDAS: 573 return 574 575 # Pandas DataFrame natually supports 1 dim data only. 576 label_dimension = 1 577 input_dimension = label_dimension 578 batch_size = 10 579 data = np.array([1., 2., 3., 4.], dtype=np.float32) 580 x = pd.DataFrame({'x': data}) 581 y = pd.Series(data) 582 prediction_length = 4 583 584 train_input_fn = pandas_io.pandas_input_fn( 585 x=x, y=y, batch_size=batch_size, num_epochs=None, shuffle=True) 586 eval_input_fn = pandas_io.pandas_input_fn( 587 x=x, y=y, batch_size=batch_size, shuffle=False) 588 predict_input_fn = pandas_io.pandas_input_fn( 589 x=x, batch_size=batch_size, shuffle=False) 590 591 self._test_complete_flow( 592 train_input_fn=train_input_fn, 593 eval_input_fn=eval_input_fn, 594 predict_input_fn=predict_input_fn, 595 input_dimension=input_dimension, 596 label_dimension=label_dimension, 597 prediction_length=prediction_length) 598 599 def test_input_fn_from_parse_example(self): 600 """Tests complete flow with input_fn constructed from parse_example.""" 601 label_dimension = 2 602 input_dimension = label_dimension 603 batch_size = 10 604 prediction_length = batch_size 605 data = np.linspace(0., 2., batch_size * label_dimension, dtype=np.float32) 606 data = data.reshape(batch_size, label_dimension) 607 608 serialized_examples = [] 609 for datum in data: 610 example = example_pb2.Example(features=feature_pb2.Features( 611 feature={ 612 'x': 613 feature_pb2.Feature(float_list=feature_pb2.FloatList( 614 value=datum)), 615 'y': 616 feature_pb2.Feature(float_list=feature_pb2.FloatList( 617 value=datum[:label_dimension])), 618 })) 619 serialized_examples.append(example.SerializeToString()) 620 621 feature_spec = { 622 'x': parsing_ops.FixedLenFeature([input_dimension], dtypes.float32), 623 'y': parsing_ops.FixedLenFeature([label_dimension], dtypes.float32), 624 } 625 626 def _train_input_fn(): 627 feature_map = parsing_ops.parse_example(serialized_examples, feature_spec) 628 features = queue_parsed_features(feature_map) 629 labels = features.pop('y') 630 return features, labels 631 632 def _eval_input_fn(): 633 feature_map = parsing_ops.parse_example( 634 input_lib.limit_epochs(serialized_examples, num_epochs=1), 635 feature_spec) 636 features = queue_parsed_features(feature_map) 637 labels = features.pop('y') 638 return features, labels 639 640 def _predict_input_fn(): 641 feature_map = parsing_ops.parse_example( 642 input_lib.limit_epochs(serialized_examples, num_epochs=1), 643 feature_spec) 644 features = queue_parsed_features(feature_map) 645 features.pop('y') 646 return features, None 647 648 self._test_complete_flow( 649 train_input_fn=_train_input_fn, 650 eval_input_fn=_eval_input_fn, 651 predict_input_fn=_predict_input_fn, 652 input_dimension=input_dimension, 653 label_dimension=label_dimension, 654 prediction_length=prediction_length) 655 656 657 class BaseLinearRegressorTrainingTest(object): 658 659 def __init__(self, linear_regressor_fn): 660 self._linear_regressor_fn = linear_regressor_fn 661 662 def setUp(self): 663 self._model_dir = tempfile.mkdtemp() 664 665 def tearDown(self): 666 if self._model_dir: 667 writer_cache.FileWriterCache.clear() 668 shutil.rmtree(self._model_dir) 669 670 def _mock_optimizer(self, expected_loss=None): 671 expected_var_names = [ 672 '%s/part_0:0' % AGE_WEIGHT_NAME, 673 '%s/part_0:0' % BIAS_NAME 674 ] 675 676 def _minimize(loss, global_step=None, var_list=None): 677 trainable_vars = var_list or ops.get_collection( 678 ops.GraphKeys.TRAINABLE_VARIABLES) 679 self.assertItemsEqual(expected_var_names, 680 [var.name for var in trainable_vars]) 681 682 # Verify loss. We can't check the value directly, so we add an assert op. 683 self.assertEquals(0, loss.shape.ndims) 684 if expected_loss is None: 685 if global_step is not None: 686 return state_ops.assign_add(global_step, 1).op 687 return control_flow_ops.no_op() 688 assert_loss = assert_close( 689 math_ops.to_float(expected_loss, name='expected'), 690 loss, 691 name='assert_loss') 692 with ops.control_dependencies((assert_loss,)): 693 if global_step is not None: 694 return state_ops.assign_add(global_step, 1).op 695 return control_flow_ops.no_op() 696 697 mock_optimizer = test.mock.NonCallableMock( 698 spec=optimizer_lib.Optimizer, 699 wraps=optimizer_lib.Optimizer(use_locking=False, name='my_optimizer')) 700 mock_optimizer.minimize = test.mock.MagicMock(wraps=_minimize) 701 702 # NOTE: Estimator.params performs a deepcopy, which wreaks havoc with mocks. 703 # So, return mock_optimizer itself for deepcopy. 704 mock_optimizer.__deepcopy__ = lambda _: mock_optimizer 705 return mock_optimizer 706 707 def _assert_checkpoint(self, 708 expected_global_step, 709 expected_age_weight=None, 710 expected_bias=None): 711 shapes = { 712 name: shape 713 for (name, shape) in checkpoint_utils.list_variables(self._model_dir) 714 } 715 716 self.assertEqual([], shapes[ops.GraphKeys.GLOBAL_STEP]) 717 self.assertEqual(expected_global_step, 718 checkpoint_utils.load_variable(self._model_dir, 719 ops.GraphKeys.GLOBAL_STEP)) 720 721 self.assertEqual([1, 1], shapes[AGE_WEIGHT_NAME]) 722 if expected_age_weight is not None: 723 self.assertEqual(expected_age_weight, 724 checkpoint_utils.load_variable(self._model_dir, 725 AGE_WEIGHT_NAME)) 726 727 self.assertEqual([1], shapes[BIAS_NAME]) 728 if expected_bias is not None: 729 self.assertEqual(expected_bias, 730 checkpoint_utils.load_variable(self._model_dir, 731 BIAS_NAME)) 732 733 def testFromScratchWithDefaultOptimizer(self): 734 # Create LinearRegressor. 735 label = 5. 736 age = 17 737 linear_regressor = self._linear_regressor_fn( 738 feature_columns=(feature_column_lib.numeric_column('age'),), 739 model_dir=self._model_dir) 740 741 # Train for a few steps, and validate final checkpoint. 742 num_steps = 10 743 linear_regressor.train( 744 input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps) 745 self._assert_checkpoint(num_steps) 746 747 def testTrainWithOneDimLabel(self): 748 label_dimension = 1 749 batch_size = 20 750 feature_columns = [feature_column_lib.numeric_column('age', shape=(1,))] 751 est = self._linear_regressor_fn( 752 feature_columns=feature_columns, 753 label_dimension=label_dimension, 754 model_dir=self._model_dir) 755 data_rank_1 = np.linspace(0., 2., batch_size, dtype=np.float32) 756 self.assertEqual((batch_size,), data_rank_1.shape) 757 758 train_input_fn = numpy_io.numpy_input_fn( 759 x={'age': data_rank_1}, 760 y=data_rank_1, 761 batch_size=batch_size, 762 num_epochs=None, 763 shuffle=True) 764 est.train(train_input_fn, steps=200) 765 self._assert_checkpoint(200) 766 767 def testTrainWithOneDimWeight(self): 768 label_dimension = 1 769 batch_size = 20 770 feature_columns = [feature_column_lib.numeric_column('age', shape=(1,))] 771 est = self._linear_regressor_fn( 772 feature_columns=feature_columns, 773 label_dimension=label_dimension, 774 weight_column='w', 775 model_dir=self._model_dir) 776 777 data_rank_1 = np.linspace(0., 2., batch_size, dtype=np.float32) 778 self.assertEqual((batch_size,), data_rank_1.shape) 779 780 train_input_fn = numpy_io.numpy_input_fn( 781 x={'age': data_rank_1, 782 'w': data_rank_1}, 783 y=data_rank_1, 784 batch_size=batch_size, 785 num_epochs=None, 786 shuffle=True) 787 est.train(train_input_fn, steps=200) 788 self._assert_checkpoint(200) 789 790 def testFromScratch(self): 791 # Create LinearRegressor. 792 label = 5. 793 age = 17 794 # loss = (logits - label)^2 = (0 - 5.)^2 = 25. 795 mock_optimizer = self._mock_optimizer(expected_loss=25.) 796 linear_regressor = self._linear_regressor_fn( 797 feature_columns=(feature_column_lib.numeric_column('age'),), 798 model_dir=self._model_dir, 799 optimizer=mock_optimizer) 800 self.assertEqual(0, mock_optimizer.minimize.call_count) 801 802 # Train for a few steps, and validate optimizer and final checkpoint. 803 num_steps = 10 804 linear_regressor.train( 805 input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps) 806 self.assertEqual(1, mock_optimizer.minimize.call_count) 807 self._assert_checkpoint( 808 expected_global_step=num_steps, 809 expected_age_weight=0., 810 expected_bias=0.) 811 812 def testFromCheckpoint(self): 813 # Create initial checkpoint. 814 age_weight = 10.0 815 bias = 5.0 816 initial_global_step = 100 817 with ops.Graph().as_default(): 818 variables_lib.Variable([[age_weight]], name=AGE_WEIGHT_NAME) 819 variables_lib.Variable([bias], name=BIAS_NAME) 820 variables_lib.Variable( 821 initial_global_step, 822 name=ops.GraphKeys.GLOBAL_STEP, 823 dtype=dtypes.int64) 824 save_variables_to_ckpt(self._model_dir) 825 826 # logits = age * age_weight + bias = 17 * 10. + 5. = 175 827 # loss = (logits - label)^2 = (175 - 5)^2 = 28900 828 mock_optimizer = self._mock_optimizer(expected_loss=28900.) 829 linear_regressor = self._linear_regressor_fn( 830 feature_columns=(feature_column_lib.numeric_column('age'),), 831 model_dir=self._model_dir, 832 optimizer=mock_optimizer) 833 self.assertEqual(0, mock_optimizer.minimize.call_count) 834 835 # Train for a few steps, and validate optimizer and final checkpoint. 836 num_steps = 10 837 linear_regressor.train( 838 input_fn=lambda: ({'age': ((17,),)}, ((5.,),)), steps=num_steps) 839 self.assertEqual(1, mock_optimizer.minimize.call_count) 840 self._assert_checkpoint( 841 expected_global_step=initial_global_step + num_steps, 842 expected_age_weight=age_weight, 843 expected_bias=bias) 844 845 def testFromCheckpointMultiBatch(self): 846 # Create initial checkpoint. 847 age_weight = 10.0 848 bias = 5.0 849 initial_global_step = 100 850 with ops.Graph().as_default(): 851 variables_lib.Variable([[age_weight]], name=AGE_WEIGHT_NAME) 852 variables_lib.Variable([bias], name=BIAS_NAME) 853 variables_lib.Variable( 854 initial_global_step, 855 name=ops.GraphKeys.GLOBAL_STEP, 856 dtype=dtypes.int64) 857 save_variables_to_ckpt(self._model_dir) 858 859 # logits = age * age_weight + bias 860 # logits[0] = 17 * 10. + 5. = 175 861 # logits[1] = 15 * 10. + 5. = 155 862 # loss = sum(logits - label)^2 = (175 - 5)^2 + (155 - 3)^2 = 52004 863 mock_optimizer = self._mock_optimizer(expected_loss=52004.) 864 linear_regressor = self._linear_regressor_fn( 865 feature_columns=(feature_column_lib.numeric_column('age'),), 866 model_dir=self._model_dir, 867 optimizer=mock_optimizer) 868 self.assertEqual(0, mock_optimizer.minimize.call_count) 869 870 # Train for a few steps, and validate optimizer and final checkpoint. 871 num_steps = 10 872 linear_regressor.train( 873 input_fn=lambda: ({'age': ((17,), (15,))}, ((5.,), (3.,))), 874 steps=num_steps) 875 self.assertEqual(1, mock_optimizer.minimize.call_count) 876 self._assert_checkpoint( 877 expected_global_step=initial_global_step + num_steps, 878 expected_age_weight=age_weight, 879 expected_bias=bias) 880 881 882 class BaseLinearClassifierTrainingTest(object): 883 884 def __init__(self, linear_classifier_fn): 885 self._linear_classifier_fn = linear_classifier_fn 886 887 def setUp(self): 888 self._model_dir = tempfile.mkdtemp() 889 890 def tearDown(self): 891 if self._model_dir: 892 shutil.rmtree(self._model_dir) 893 894 def _mock_optimizer(self, expected_loss=None): 895 expected_var_names = [ 896 '%s/part_0:0' % AGE_WEIGHT_NAME, 897 '%s/part_0:0' % BIAS_NAME 898 ] 899 900 def _minimize(loss, global_step): 901 trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) 902 self.assertItemsEqual( 903 expected_var_names, 904 [var.name for var in trainable_vars]) 905 906 # Verify loss. We can't check the value directly, so we add an assert op. 907 self.assertEquals(0, loss.shape.ndims) 908 if expected_loss is None: 909 return state_ops.assign_add(global_step, 1).op 910 assert_loss = assert_close( 911 math_ops.to_float(expected_loss, name='expected'), 912 loss, 913 name='assert_loss') 914 with ops.control_dependencies((assert_loss,)): 915 return state_ops.assign_add(global_step, 1).op 916 917 mock_optimizer = test.mock.NonCallableMock( 918 spec=optimizer_lib.Optimizer, 919 wraps=optimizer_lib.Optimizer(use_locking=False, name='my_optimizer')) 920 mock_optimizer.minimize = test.mock.MagicMock(wraps=_minimize) 921 922 # NOTE: Estimator.params performs a deepcopy, which wreaks havoc with mocks. 923 # So, return mock_optimizer itself for deepcopy. 924 mock_optimizer.__deepcopy__ = lambda _: mock_optimizer 925 return mock_optimizer 926 927 def _assert_checkpoint( 928 self, n_classes, expected_global_step, expected_age_weight=None, 929 expected_bias=None): 930 logits_dimension = n_classes if n_classes > 2 else 1 931 932 shapes = { 933 name: shape for (name, shape) in 934 checkpoint_utils.list_variables(self._model_dir) 935 } 936 937 self.assertEqual([], shapes[ops.GraphKeys.GLOBAL_STEP]) 938 self.assertEqual( 939 expected_global_step, 940 checkpoint_utils.load_variable( 941 self._model_dir, ops.GraphKeys.GLOBAL_STEP)) 942 943 self.assertEqual([1, logits_dimension], 944 shapes[AGE_WEIGHT_NAME]) 945 if expected_age_weight is not None: 946 self.assertAllEqual(expected_age_weight, 947 checkpoint_utils.load_variable( 948 self._model_dir, 949 AGE_WEIGHT_NAME)) 950 951 self.assertEqual([logits_dimension], shapes[BIAS_NAME]) 952 if expected_bias is not None: 953 self.assertAllEqual(expected_bias, 954 checkpoint_utils.load_variable( 955 self._model_dir, BIAS_NAME)) 956 957 def _testFromScratchWithDefaultOptimizer(self, n_classes): 958 label = 0 959 age = 17 960 est = linear.LinearClassifier( 961 feature_columns=(feature_column_lib.numeric_column('age'),), 962 n_classes=n_classes, 963 model_dir=self._model_dir) 964 965 # Train for a few steps, and validate final checkpoint. 966 num_steps = 10 967 est.train( 968 input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps) 969 self._assert_checkpoint(n_classes, num_steps) 970 971 def testBinaryClassesFromScratchWithDefaultOptimizer(self): 972 self._testFromScratchWithDefaultOptimizer(n_classes=2) 973 974 def testMultiClassesFromScratchWithDefaultOptimizer(self): 975 self._testFromScratchWithDefaultOptimizer(n_classes=4) 976 977 def _testTrainWithTwoDimsLabel(self, n_classes): 978 batch_size = 20 979 980 est = linear.LinearClassifier( 981 feature_columns=(feature_column_lib.numeric_column('age'),), 982 n_classes=n_classes, 983 model_dir=self._model_dir) 984 data_rank_1 = np.array([0, 1]) 985 data_rank_2 = np.array([[0], [1]]) 986 self.assertEqual((2,), data_rank_1.shape) 987 self.assertEqual((2, 1), data_rank_2.shape) 988 989 train_input_fn = numpy_io.numpy_input_fn( 990 x={'age': data_rank_1}, 991 y=data_rank_2, 992 batch_size=batch_size, 993 num_epochs=None, 994 shuffle=True) 995 est.train(train_input_fn, steps=200) 996 self._assert_checkpoint(n_classes, 200) 997 998 def testBinaryClassesTrainWithTwoDimsLabel(self): 999 self._testTrainWithTwoDimsLabel(n_classes=2) 1000 1001 def testMultiClassesTrainWithTwoDimsLabel(self): 1002 self._testTrainWithTwoDimsLabel(n_classes=4) 1003 1004 def _testTrainWithOneDimLabel(self, n_classes): 1005 batch_size = 20 1006 1007 est = linear.LinearClassifier( 1008 feature_columns=(feature_column_lib.numeric_column('age'),), 1009 n_classes=n_classes, 1010 model_dir=self._model_dir) 1011 data_rank_1 = np.array([0, 1]) 1012 self.assertEqual((2,), data_rank_1.shape) 1013 1014 train_input_fn = numpy_io.numpy_input_fn( 1015 x={'age': data_rank_1}, 1016 y=data_rank_1, 1017 batch_size=batch_size, 1018 num_epochs=None, 1019 shuffle=True) 1020 est.train(train_input_fn, steps=200) 1021 self._assert_checkpoint(n_classes, 200) 1022 1023 def testBinaryClassesTrainWithOneDimLabel(self): 1024 self._testTrainWithOneDimLabel(n_classes=2) 1025 1026 def testMultiClassesTrainWithOneDimLabel(self): 1027 self._testTrainWithOneDimLabel(n_classes=4) 1028 1029 def _testTrainWithTwoDimsWeight(self, n_classes): 1030 batch_size = 20 1031 1032 est = linear.LinearClassifier( 1033 feature_columns=(feature_column_lib.numeric_column('age'),), 1034 weight_column='w', 1035 n_classes=n_classes, 1036 model_dir=self._model_dir) 1037 data_rank_1 = np.array([0, 1]) 1038 data_rank_2 = np.array([[0], [1]]) 1039 self.assertEqual((2,), data_rank_1.shape) 1040 self.assertEqual((2, 1), data_rank_2.shape) 1041 1042 train_input_fn = numpy_io.numpy_input_fn( 1043 x={'age': data_rank_1, 'w': data_rank_2}, y=data_rank_1, 1044 batch_size=batch_size, num_epochs=None, 1045 shuffle=True) 1046 est.train(train_input_fn, steps=200) 1047 self._assert_checkpoint(n_classes, 200) 1048 1049 def testBinaryClassesTrainWithTwoDimsWeight(self): 1050 self._testTrainWithTwoDimsWeight(n_classes=2) 1051 1052 def testMultiClassesTrainWithTwoDimsWeight(self): 1053 self._testTrainWithTwoDimsWeight(n_classes=4) 1054 1055 def _testTrainWithOneDimWeight(self, n_classes): 1056 batch_size = 20 1057 1058 est = linear.LinearClassifier( 1059 feature_columns=(feature_column_lib.numeric_column('age'),), 1060 weight_column='w', 1061 n_classes=n_classes, 1062 model_dir=self._model_dir) 1063 data_rank_1 = np.array([0, 1]) 1064 self.assertEqual((2,), data_rank_1.shape) 1065 1066 train_input_fn = numpy_io.numpy_input_fn( 1067 x={'age': data_rank_1, 'w': data_rank_1}, y=data_rank_1, 1068 batch_size=batch_size, num_epochs=None, 1069 shuffle=True) 1070 est.train(train_input_fn, steps=200) 1071 self._assert_checkpoint(n_classes, 200) 1072 1073 def testBinaryClassesTrainWithOneDimWeight(self): 1074 self._testTrainWithOneDimWeight(n_classes=2) 1075 1076 def testMultiClassesTrainWithOneDimWeight(self): 1077 self._testTrainWithOneDimWeight(n_classes=4) 1078 1079 def _testFromScratch(self, n_classes): 1080 label = 1 1081 age = 17 1082 # For binary classifier: 1083 # loss = sigmoid_cross_entropy(logits, label) where logits=0 (weights are 1084 # all zero initially) and label = 1 so, 1085 # loss = 1 * -log ( sigmoid(logits) ) = 0.69315 1086 # For multi class classifier: 1087 # loss = cross_entropy(logits, label) where logits are all 0s (weights are 1088 # all zero initially) and label = 1 so, 1089 # loss = 1 * -log ( 1.0 / n_classes ) 1090 # For this particular test case, as logits are same, the formular 1091 # 1 * -log ( 1.0 / n_classes ) covers both binary and multi class cases. 1092 mock_optimizer = self._mock_optimizer( 1093 expected_loss=-1 * math.log(1.0/n_classes)) 1094 1095 est = linear.LinearClassifier( 1096 feature_columns=(feature_column_lib.numeric_column('age'),), 1097 n_classes=n_classes, 1098 optimizer=mock_optimizer, 1099 model_dir=self._model_dir) 1100 self.assertEqual(0, mock_optimizer.minimize.call_count) 1101 1102 # Train for a few steps, and validate optimizer and final checkpoint. 1103 num_steps = 10 1104 est.train( 1105 input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps) 1106 self.assertEqual(1, mock_optimizer.minimize.call_count) 1107 self._assert_checkpoint( 1108 n_classes, 1109 expected_global_step=num_steps, 1110 expected_age_weight=[[0.]] if n_classes == 2 else [[0.] * n_classes], 1111 expected_bias=[0.] if n_classes == 2 else [.0] * n_classes) 1112 1113 def testBinaryClassesFromScratch(self): 1114 self._testFromScratch(n_classes=2) 1115 1116 def testMultiClassesFromScratch(self): 1117 self._testFromScratch(n_classes=4) 1118 1119 def _testFromCheckpoint(self, n_classes): 1120 # Create initial checkpoint. 1121 label = 1 1122 age = 17 1123 # For binary case, the expected weight has shape (1,1). For multi class 1124 # case, the shape is (1, n_classes). In order to test the weights, set 1125 # weights as 2.0 * range(n_classes). 1126 age_weight = [[2.0]] if n_classes == 2 else ( 1127 np.reshape(2.0 * np.array(list(range(n_classes)), dtype=np.float32), 1128 (1, n_classes))) 1129 bias = [-35.0] if n_classes == 2 else [-35.0] * n_classes 1130 initial_global_step = 100 1131 with ops.Graph().as_default(): 1132 variables_lib.Variable(age_weight, name=AGE_WEIGHT_NAME) 1133 variables_lib.Variable(bias, name=BIAS_NAME) 1134 variables_lib.Variable( 1135 initial_global_step, 1136 name=ops.GraphKeys.GLOBAL_STEP, 1137 dtype=dtypes.int64) 1138 save_variables_to_ckpt(self._model_dir) 1139 1140 # For binary classifier: 1141 # logits = age * age_weight + bias = 17 * 2. - 35. = -1. 1142 # loss = sigmoid_cross_entropy(logits, label) 1143 # so, loss = 1 * -log ( sigmoid(-1) ) = 1.3133 1144 # For multi class classifier: 1145 # loss = cross_entropy(logits, label) 1146 # where logits = 17 * age_weight + bias and label = 1 1147 # so, loss = 1 * -log ( soft_max(logits)[1] ) 1148 if n_classes == 2: 1149 expected_loss = 1.3133 1150 else: 1151 logits = age_weight * age + bias 1152 logits_exp = np.exp(logits) 1153 softmax = logits_exp / logits_exp.sum() 1154 expected_loss = -1 * math.log(softmax[0, label]) 1155 1156 mock_optimizer = self._mock_optimizer(expected_loss=expected_loss) 1157 1158 est = linear.LinearClassifier( 1159 feature_columns=(feature_column_lib.numeric_column('age'),), 1160 n_classes=n_classes, 1161 optimizer=mock_optimizer, 1162 model_dir=self._model_dir) 1163 self.assertEqual(0, mock_optimizer.minimize.call_count) 1164 1165 # Train for a few steps, and validate optimizer and final checkpoint. 1166 num_steps = 10 1167 est.train( 1168 input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps) 1169 self.assertEqual(1, mock_optimizer.minimize.call_count) 1170 self._assert_checkpoint( 1171 n_classes, 1172 expected_global_step=initial_global_step + num_steps, 1173 expected_age_weight=age_weight, 1174 expected_bias=bias) 1175 1176 def testBinaryClassesFromCheckpoint(self): 1177 self._testFromCheckpoint(n_classes=2) 1178 1179 def testMultiClassesFromCheckpoint(self): 1180 self._testFromCheckpoint(n_classes=4) 1181 1182 def _testFromCheckpointFloatLabels(self, n_classes): 1183 """Tests float labels for binary classification.""" 1184 # Create initial checkpoint. 1185 if n_classes > 2: 1186 return 1187 label = 0.8 1188 age = 17 1189 age_weight = [[2.0]] 1190 bias = [-35.0] 1191 initial_global_step = 100 1192 with ops.Graph().as_default(): 1193 variables_lib.Variable(age_weight, name=AGE_WEIGHT_NAME) 1194 variables_lib.Variable(bias, name=BIAS_NAME) 1195 variables_lib.Variable( 1196 initial_global_step, 1197 name=ops.GraphKeys.GLOBAL_STEP, 1198 dtype=dtypes.int64) 1199 save_variables_to_ckpt(self._model_dir) 1200 1201 # logits = age * age_weight + bias = 17 * 2. - 35. = -1. 1202 # loss = sigmoid_cross_entropy(logits, label) 1203 # => loss = -0.8 * log(sigmoid(-1)) -0.2 * log(sigmoid(+1)) = 1.1132617 1204 mock_optimizer = self._mock_optimizer(expected_loss=1.1132617) 1205 1206 est = linear.LinearClassifier( 1207 feature_columns=(feature_column_lib.numeric_column('age'),), 1208 n_classes=n_classes, 1209 optimizer=mock_optimizer, 1210 model_dir=self._model_dir) 1211 self.assertEqual(0, mock_optimizer.minimize.call_count) 1212 1213 # Train for a few steps, and validate optimizer and final checkpoint. 1214 num_steps = 10 1215 est.train( 1216 input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps) 1217 self.assertEqual(1, mock_optimizer.minimize.call_count) 1218 1219 def testBinaryClassesFromCheckpointFloatLabels(self): 1220 self._testFromCheckpointFloatLabels(n_classes=2) 1221 1222 def testMultiClassesFromCheckpointFloatLabels(self): 1223 self._testFromCheckpointFloatLabels(n_classes=4) 1224 1225 def _testFromCheckpointMultiBatch(self, n_classes): 1226 # Create initial checkpoint. 1227 label = [1, 0] 1228 age = [17, 18.5] 1229 # For binary case, the expected weight has shape (1,1). For multi class 1230 # case, the shape is (1, n_classes). In order to test the weights, set 1231 # weights as 2.0 * range(n_classes). 1232 age_weight = [[2.0]] if n_classes == 2 else ( 1233 np.reshape(2.0 * np.array(list(range(n_classes)), dtype=np.float32), 1234 (1, n_classes))) 1235 bias = [-35.0] if n_classes == 2 else [-35.0] * n_classes 1236 initial_global_step = 100 1237 with ops.Graph().as_default(): 1238 variables_lib.Variable(age_weight, name=AGE_WEIGHT_NAME) 1239 variables_lib.Variable(bias, name=BIAS_NAME) 1240 variables_lib.Variable( 1241 initial_global_step, 1242 name=ops.GraphKeys.GLOBAL_STEP, 1243 dtype=dtypes.int64) 1244 save_variables_to_ckpt(self._model_dir) 1245 1246 # For binary classifier: 1247 # logits = age * age_weight + bias 1248 # logits[0] = 17 * 2. - 35. = -1. 1249 # logits[1] = 18.5 * 2. - 35. = 2. 1250 # loss = sigmoid_cross_entropy(logits, label) 1251 # so, loss[0] = 1 * -log ( sigmoid(-1) ) = 1.3133 1252 # loss[1] = (1 - 0) * -log ( 1- sigmoid(2) ) = 2.1269 1253 # For multi class classifier: 1254 # loss = cross_entropy(logits, label) 1255 # where logits = [17, 18.5] * age_weight + bias and label = [1, 0] 1256 # so, loss = 1 * -log ( soft_max(logits)[label] ) 1257 if n_classes == 2: 1258 expected_loss = (1.3133 + 2.1269) 1259 else: 1260 logits = age_weight * np.reshape(age, (2, 1)) + bias 1261 logits_exp = np.exp(logits) 1262 softmax_row_0 = logits_exp[0] / logits_exp[0].sum() 1263 softmax_row_1 = logits_exp[1] / logits_exp[1].sum() 1264 expected_loss_0 = -1 * math.log(softmax_row_0[label[0]]) 1265 expected_loss_1 = -1 * math.log(softmax_row_1[label[1]]) 1266 expected_loss = expected_loss_0 + expected_loss_1 1267 1268 mock_optimizer = self._mock_optimizer(expected_loss=expected_loss) 1269 1270 est = linear.LinearClassifier( 1271 feature_columns=(feature_column_lib.numeric_column('age'),), 1272 n_classes=n_classes, 1273 optimizer=mock_optimizer, 1274 model_dir=self._model_dir) 1275 self.assertEqual(0, mock_optimizer.minimize.call_count) 1276 1277 # Train for a few steps, and validate optimizer and final checkpoint. 1278 num_steps = 10 1279 est.train( 1280 input_fn=lambda: ({'age': (age)}, (label)), 1281 steps=num_steps) 1282 self.assertEqual(1, mock_optimizer.minimize.call_count) 1283 self._assert_checkpoint( 1284 n_classes, 1285 expected_global_step=initial_global_step + num_steps, 1286 expected_age_weight=age_weight, 1287 expected_bias=bias) 1288 1289 def testBinaryClassesFromCheckpointMultiBatch(self): 1290 self._testFromCheckpointMultiBatch(n_classes=2) 1291 1292 def testMultiClassesFromCheckpointMultiBatch(self): 1293 self._testFromCheckpointMultiBatch(n_classes=4) 1294 1295 1296 class BaseLinearClassifierEvaluationTest(object): 1297 1298 def __init__(self, linear_classifier_fn): 1299 self._linear_classifier_fn = linear_classifier_fn 1300 1301 def setUp(self): 1302 self._model_dir = tempfile.mkdtemp() 1303 1304 def tearDown(self): 1305 if self._model_dir: 1306 shutil.rmtree(self._model_dir) 1307 1308 def _test_evaluation_for_simple_data(self, n_classes): 1309 label = 1 1310 age = 1. 1311 1312 # For binary case, the expected weight has shape (1,1). For multi class 1313 # case, the shape is (1, n_classes). In order to test the weights, set 1314 # weights as 2.0 * range(n_classes). 1315 age_weight = [[-11.0]] if n_classes == 2 else ( 1316 np.reshape(-11.0 * np.array(list(range(n_classes)), dtype=np.float32), 1317 (1, n_classes))) 1318 bias = [-30.0] if n_classes == 2 else [-30.0] * n_classes 1319 1320 with ops.Graph().as_default(): 1321 variables_lib.Variable(age_weight, name=AGE_WEIGHT_NAME) 1322 variables_lib.Variable(bias, name=BIAS_NAME) 1323 variables_lib.Variable( 1324 100, name=ops.GraphKeys.GLOBAL_STEP, dtype=dtypes.int64) 1325 save_variables_to_ckpt(self._model_dir) 1326 1327 est = self._linear_classifier_fn( 1328 feature_columns=(feature_column_lib.numeric_column('age'),), 1329 n_classes=n_classes, 1330 model_dir=self._model_dir) 1331 eval_metrics = est.evaluate( 1332 input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=1) 1333 1334 if n_classes == 2: 1335 # Binary classes: loss = sum(corss_entropy(41)) = 41. 1336 expected_metrics = { 1337 metric_keys.MetricKeys.LOSS: 41., 1338 ops.GraphKeys.GLOBAL_STEP: 100, 1339 metric_keys.MetricKeys.LOSS_MEAN: 41., 1340 metric_keys.MetricKeys.ACCURACY: 0., 1341 metric_keys.MetricKeys.PREDICTION_MEAN: 0., 1342 metric_keys.MetricKeys.LABEL_MEAN: 1., 1343 metric_keys.MetricKeys.ACCURACY_BASELINE: 1, 1344 metric_keys.MetricKeys.AUC: 0., 1345 metric_keys.MetricKeys.AUC_PR: 0.5, 1346 } 1347 else: 1348 # Multi classes: loss = 1 * -log ( soft_max(logits)[label] ) 1349 logits = age_weight * age + bias 1350 logits_exp = np.exp(logits) 1351 softmax = logits_exp / logits_exp.sum() 1352 expected_loss = -1 * math.log(softmax[0, label]) 1353 1354 expected_metrics = { 1355 metric_keys.MetricKeys.LOSS: expected_loss, 1356 ops.GraphKeys.GLOBAL_STEP: 100, 1357 metric_keys.MetricKeys.LOSS_MEAN: expected_loss, 1358 metric_keys.MetricKeys.ACCURACY: 0., 1359 } 1360 1361 self.assertAllClose(sorted_key_dict(expected_metrics), 1362 sorted_key_dict(eval_metrics), rtol=1e-3) 1363 1364 def test_binary_classes_evaluation_for_simple_data(self): 1365 self._test_evaluation_for_simple_data(n_classes=2) 1366 1367 def test_multi_classes_evaluation_for_simple_data(self): 1368 self._test_evaluation_for_simple_data(n_classes=4) 1369 1370 def _test_evaluation_batch(self, n_classes): 1371 """Tests evaluation for batch_size==2.""" 1372 label = [1, 0] 1373 age = [17., 18.] 1374 # For binary case, the expected weight has shape (1,1). For multi class 1375 # case, the shape is (1, n_classes). In order to test the weights, set 1376 # weights as 2.0 * range(n_classes). 1377 age_weight = [[2.0]] if n_classes == 2 else ( 1378 np.reshape(2.0 * np.array(list(range(n_classes)), dtype=np.float32), 1379 (1, n_classes))) 1380 bias = [-35.0] if n_classes == 2 else [-35.0] * n_classes 1381 initial_global_step = 100 1382 with ops.Graph().as_default(): 1383 variables_lib.Variable(age_weight, name=AGE_WEIGHT_NAME) 1384 variables_lib.Variable(bias, name=BIAS_NAME) 1385 variables_lib.Variable( 1386 initial_global_step, 1387 name=ops.GraphKeys.GLOBAL_STEP, 1388 dtype=dtypes.int64) 1389 save_variables_to_ckpt(self._model_dir) 1390 1391 est = self._linear_classifier_fn( 1392 feature_columns=(feature_column_lib.numeric_column('age'),), 1393 n_classes=n_classes, 1394 model_dir=self._model_dir) 1395 eval_metrics = est.evaluate( 1396 input_fn=lambda: ({'age': (age)}, (label)), steps=1) 1397 1398 if n_classes == 2: 1399 # Logits are (-1., 1.) labels are (1, 0). 1400 # Loss is 1401 # loss for row 1: 1 * -log(sigmoid(-1)) = 1.3133 1402 # loss for row 2: (1 - 0) * -log(1 - sigmoid(1)) = 1.3133 1403 expected_loss = 1.3133 * 2 1404 1405 expected_metrics = { 1406 metric_keys.MetricKeys.LOSS: expected_loss, 1407 ops.GraphKeys.GLOBAL_STEP: 100, 1408 metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 2, 1409 metric_keys.MetricKeys.ACCURACY: 0., 1410 metric_keys.MetricKeys.PREDICTION_MEAN: 0.5, 1411 metric_keys.MetricKeys.LABEL_MEAN: 0.5, 1412 metric_keys.MetricKeys.ACCURACY_BASELINE: 0.5, 1413 metric_keys.MetricKeys.AUC: 0., 1414 metric_keys.MetricKeys.AUC_PR: 0.25, 1415 } 1416 else: 1417 # Multi classes: loss = 1 * -log ( soft_max(logits)[label] ) 1418 logits = age_weight * np.reshape(age, (2, 1)) + bias 1419 logits_exp = np.exp(logits) 1420 softmax_row_0 = logits_exp[0] / logits_exp[0].sum() 1421 softmax_row_1 = logits_exp[1] / logits_exp[1].sum() 1422 expected_loss_0 = -1 * math.log(softmax_row_0[label[0]]) 1423 expected_loss_1 = -1 * math.log(softmax_row_1[label[1]]) 1424 expected_loss = expected_loss_0 + expected_loss_1 1425 1426 expected_metrics = { 1427 metric_keys.MetricKeys.LOSS: expected_loss, 1428 ops.GraphKeys.GLOBAL_STEP: 100, 1429 metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 2, 1430 metric_keys.MetricKeys.ACCURACY: 0., 1431 } 1432 1433 self.assertAllClose(sorted_key_dict(expected_metrics), 1434 sorted_key_dict(eval_metrics), rtol=1e-3) 1435 1436 def test_binary_classes_evaluation_batch(self): 1437 self._test_evaluation_batch(n_classes=2) 1438 1439 def test_multi_classes_evaluation_batch(self): 1440 self._test_evaluation_batch(n_classes=4) 1441 1442 def _test_evaluation_weights(self, n_classes): 1443 """Tests evaluation with weights.""" 1444 1445 label = [1, 0] 1446 age = [17., 18.] 1447 weights = [1., 2.] 1448 # For binary case, the expected weight has shape (1,1). For multi class 1449 # case, the shape is (1, n_classes). In order to test the weights, set 1450 # weights as 2.0 * range(n_classes). 1451 age_weight = [[2.0]] if n_classes == 2 else ( 1452 np.reshape(2.0 * np.array(list(range(n_classes)), dtype=np.float32), 1453 (1, n_classes))) 1454 bias = [-35.0] if n_classes == 2 else [-35.0] * n_classes 1455 initial_global_step = 100 1456 with ops.Graph().as_default(): 1457 variables_lib.Variable(age_weight, name=AGE_WEIGHT_NAME) 1458 variables_lib.Variable(bias, name=BIAS_NAME) 1459 variables_lib.Variable( 1460 initial_global_step, 1461 name=ops.GraphKeys.GLOBAL_STEP, 1462 dtype=dtypes.int64) 1463 save_variables_to_ckpt(self._model_dir) 1464 1465 est = self._linear_classifier_fn( 1466 feature_columns=(feature_column_lib.numeric_column('age'),), 1467 n_classes=n_classes, 1468 weight_column='w', 1469 model_dir=self._model_dir) 1470 eval_metrics = est.evaluate( 1471 input_fn=lambda: ({'age': (age), 'w': (weights)}, (label)), steps=1) 1472 1473 if n_classes == 2: 1474 # Logits are (-1., 1.) labels are (1, 0). 1475 # Loss is 1476 # loss for row 1: 1 * -log(sigmoid(-1)) = 1.3133 1477 # loss for row 2: (1 - 0) * -log(1 - sigmoid(1)) = 1.3133 1478 # weights = [1., 2.] 1479 expected_loss = 1.3133 * (1. + 2.) 1480 loss_mean = expected_loss / (1.0 + 2.0) 1481 label_mean = np.average(label, weights=weights) 1482 logits = [-1, 1] 1483 logistics = sigmoid(np.array(logits)) 1484 predictions_mean = np.average(logistics, weights=weights) 1485 1486 expected_metrics = { 1487 metric_keys.MetricKeys.LOSS: expected_loss, 1488 ops.GraphKeys.GLOBAL_STEP: 100, 1489 metric_keys.MetricKeys.LOSS_MEAN: loss_mean, 1490 metric_keys.MetricKeys.ACCURACY: 0., 1491 metric_keys.MetricKeys.PREDICTION_MEAN: predictions_mean, 1492 metric_keys.MetricKeys.LABEL_MEAN: label_mean, 1493 metric_keys.MetricKeys.ACCURACY_BASELINE: ( 1494 max(label_mean, 1-label_mean)), 1495 metric_keys.MetricKeys.AUC: 0., 1496 metric_keys.MetricKeys.AUC_PR: 0.1668, 1497 } 1498 else: 1499 # Multi classes: unweighted_loss = 1 * -log ( soft_max(logits)[label] ) 1500 logits = age_weight * np.reshape(age, (2, 1)) + bias 1501 logits_exp = np.exp(logits) 1502 softmax_row_0 = logits_exp[0] / logits_exp[0].sum() 1503 softmax_row_1 = logits_exp[1] / logits_exp[1].sum() 1504 expected_loss_0 = -1 * math.log(softmax_row_0[label[0]]) 1505 expected_loss_1 = -1 * math.log(softmax_row_1[label[1]]) 1506 loss_mean = np.average([expected_loss_0, expected_loss_1], 1507 weights=weights) 1508 expected_loss = loss_mean * np.sum(weights) 1509 1510 expected_metrics = { 1511 metric_keys.MetricKeys.LOSS: expected_loss, 1512 ops.GraphKeys.GLOBAL_STEP: 100, 1513 metric_keys.MetricKeys.LOSS_MEAN: loss_mean, 1514 metric_keys.MetricKeys.ACCURACY: 0., 1515 } 1516 1517 self.assertAllClose(sorted_key_dict(expected_metrics), 1518 sorted_key_dict(eval_metrics), rtol=1e-3) 1519 1520 def test_binary_classes_evaluation_weights(self): 1521 self._test_evaluation_weights(n_classes=2) 1522 1523 def test_multi_classes_evaluation_weights(self): 1524 self._test_evaluation_weights(n_classes=4) 1525 1526 1527 class BaseLinearClassifierPredictTest(object): 1528 1529 def __init__(self, linear_classifier_fn): 1530 self._linear_classifier_fn = linear_classifier_fn 1531 1532 def setUp(self): 1533 self._model_dir = tempfile.mkdtemp() 1534 1535 def tearDown(self): 1536 if self._model_dir: 1537 shutil.rmtree(self._model_dir) 1538 1539 def _testPredictions(self, n_classes, label_vocabulary, label_output_fn): 1540 """Tests predict when all variables are one-dimensional.""" 1541 age = 1. 1542 1543 # For binary case, the expected weight has shape (1,1). For multi class 1544 # case, the shape is (1, n_classes). In order to test the weights, set 1545 # weights as 2.0 * range(n_classes). 1546 age_weight = [[-11.0]] if n_classes == 2 else ( 1547 np.reshape(-11.0 * np.array(list(range(n_classes)), dtype=np.float32), 1548 (1, n_classes))) 1549 bias = [10.0] if n_classes == 2 else [10.0] * n_classes 1550 1551 with ops.Graph().as_default(): 1552 variables_lib.Variable(age_weight, name=AGE_WEIGHT_NAME) 1553 variables_lib.Variable(bias, name=BIAS_NAME) 1554 variables_lib.Variable(100, name='global_step', dtype=dtypes.int64) 1555 save_variables_to_ckpt(self._model_dir) 1556 1557 est = self._linear_classifier_fn( 1558 feature_columns=(feature_column_lib.numeric_column('age'),), 1559 label_vocabulary=label_vocabulary, 1560 n_classes=n_classes, 1561 model_dir=self._model_dir) 1562 1563 predict_input_fn = numpy_io.numpy_input_fn( 1564 x={'age': np.array([[age]])}, 1565 y=None, 1566 batch_size=1, 1567 num_epochs=1, 1568 shuffle=False) 1569 predictions = list(est.predict(input_fn=predict_input_fn)) 1570 1571 if n_classes == 2: 1572 scalar_logits = np.asscalar( 1573 np.reshape(np.array(age_weight) * age + bias, (1,))) 1574 two_classes_logits = [0, scalar_logits] 1575 two_classes_logits_exp = np.exp(two_classes_logits) 1576 softmax = two_classes_logits_exp / two_classes_logits_exp.sum() 1577 1578 expected_predictions = { 1579 'class_ids': [0], 1580 'classes': [label_output_fn(0)], 1581 'logistic': [sigmoid(np.array(scalar_logits))], 1582 'logits': [scalar_logits], 1583 'probabilities': softmax, 1584 } 1585 else: 1586 onedim_logits = np.reshape(np.array(age_weight) * age + bias, (-1,)) 1587 class_ids = onedim_logits.argmax() 1588 logits_exp = np.exp(onedim_logits) 1589 softmax = logits_exp / logits_exp.sum() 1590 expected_predictions = { 1591 'class_ids': [class_ids], 1592 'classes': [label_output_fn(class_ids)], 1593 'logits': onedim_logits, 1594 'probabilities': softmax, 1595 } 1596 1597 self.assertEqual(1, len(predictions)) 1598 # assertAllClose cannot handle byte type. 1599 self.assertEqual(expected_predictions['classes'], predictions[0]['classes']) 1600 expected_predictions.pop('classes') 1601 predictions[0].pop('classes') 1602 self.assertAllClose(sorted_key_dict(expected_predictions), 1603 sorted_key_dict(predictions[0])) 1604 1605 def testBinaryClassesWithoutLabelVocabulary(self): 1606 n_classes = 2 1607 self._testPredictions(n_classes, 1608 label_vocabulary=None, 1609 label_output_fn=lambda x: ('%s' % x).encode()) 1610 1611 def testBinaryClassesWithLabelVocabulary(self): 1612 n_classes = 2 1613 self._testPredictions( 1614 n_classes, 1615 label_vocabulary=['class_vocab_{}'.format(i) 1616 for i in range(n_classes)], 1617 label_output_fn=lambda x: ('class_vocab_%s' % x).encode()) 1618 1619 def testMultiClassesWithoutLabelVocabulary(self): 1620 n_classes = 4 1621 self._testPredictions( 1622 n_classes, 1623 label_vocabulary=None, 1624 label_output_fn=lambda x: ('%s' % x).encode()) 1625 1626 def testMultiClassesWithLabelVocabulary(self): 1627 n_classes = 4 1628 self._testPredictions( 1629 n_classes, 1630 label_vocabulary=['class_vocab_{}'.format(i) 1631 for i in range(n_classes)], 1632 label_output_fn=lambda x: ('class_vocab_%s' % x).encode()) 1633 1634 1635 class BaseLinearClassifierIntegrationTest(object): 1636 1637 def __init__(self, linear_classifier_fn): 1638 self._linear_classifier_fn = linear_classifier_fn 1639 1640 def setUp(self): 1641 self._model_dir = tempfile.mkdtemp() 1642 1643 def tearDown(self): 1644 if self._model_dir: 1645 shutil.rmtree(self._model_dir) 1646 1647 def _test_complete_flow(self, n_classes, train_input_fn, eval_input_fn, 1648 predict_input_fn, input_dimension, prediction_length): 1649 feature_columns = [ 1650 feature_column_lib.numeric_column('x', shape=(input_dimension,)) 1651 ] 1652 est = self._linear_classifier_fn( 1653 feature_columns=feature_columns, 1654 n_classes=n_classes, 1655 model_dir=self._model_dir) 1656 1657 # TRAIN 1658 # learn y = x 1659 est.train(train_input_fn, steps=200) 1660 1661 # EVALUTE 1662 scores = est.evaluate(eval_input_fn) 1663 self.assertEqual(200, scores[ops.GraphKeys.GLOBAL_STEP]) 1664 self.assertIn(metric_keys.MetricKeys.LOSS, six.iterkeys(scores)) 1665 1666 # PREDICT 1667 predictions = np.array( 1668 [x['classes'] for x in est.predict(predict_input_fn)]) 1669 self.assertAllEqual((prediction_length, 1), predictions.shape) 1670 1671 # EXPORT 1672 feature_spec = feature_column_lib.make_parse_example_spec(feature_columns) 1673 serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn( 1674 feature_spec) 1675 export_dir = est.export_savedmodel(tempfile.mkdtemp(), 1676 serving_input_receiver_fn) 1677 self.assertTrue(gfile.Exists(export_dir)) 1678 1679 def _test_numpy_input_fn(self, n_classes): 1680 """Tests complete flow with numpy_input_fn.""" 1681 input_dimension = 4 1682 batch_size = 10 1683 prediction_length = batch_size 1684 data = np.linspace(0., 2., batch_size * input_dimension, dtype=np.float32) 1685 data = data.reshape(batch_size, input_dimension) 1686 target = np.array([1] * batch_size) 1687 1688 train_input_fn = numpy_io.numpy_input_fn( 1689 x={'x': data}, 1690 y=target, 1691 batch_size=batch_size, 1692 num_epochs=None, 1693 shuffle=True) 1694 eval_input_fn = numpy_io.numpy_input_fn( 1695 x={'x': data}, 1696 y=target, 1697 batch_size=batch_size, 1698 num_epochs=1, 1699 shuffle=False) 1700 predict_input_fn = numpy_io.numpy_input_fn( 1701 x={'x': data}, 1702 y=None, 1703 batch_size=batch_size, 1704 num_epochs=1, 1705 shuffle=False) 1706 1707 self._test_complete_flow( 1708 n_classes=n_classes, 1709 train_input_fn=train_input_fn, 1710 eval_input_fn=eval_input_fn, 1711 predict_input_fn=predict_input_fn, 1712 input_dimension=input_dimension, 1713 prediction_length=prediction_length) 1714 1715 def test_binary_classes_numpy_input_fn(self): 1716 self._test_numpy_input_fn(n_classes=2) 1717 1718 def test_multi_classes_numpy_input_fn(self): 1719 self._test_numpy_input_fn(n_classes=4) 1720 1721 def _test_pandas_input_fn(self, n_classes): 1722 """Tests complete flow with pandas_input_fn.""" 1723 if not HAS_PANDAS: 1724 return 1725 1726 # Pandas DataFrame natually supports 1 dim data only. 1727 input_dimension = 1 1728 batch_size = 10 1729 data = np.array([1., 2., 3., 4.], dtype=np.float32) 1730 target = np.array([1, 0, 1, 0], dtype=np.int32) 1731 x = pd.DataFrame({'x': data}) 1732 y = pd.Series(target) 1733 prediction_length = 4 1734 1735 train_input_fn = pandas_io.pandas_input_fn( 1736 x=x, y=y, batch_size=batch_size, num_epochs=None, shuffle=True) 1737 eval_input_fn = pandas_io.pandas_input_fn( 1738 x=x, y=y, batch_size=batch_size, shuffle=False) 1739 predict_input_fn = pandas_io.pandas_input_fn( 1740 x=x, batch_size=batch_size, shuffle=False) 1741 1742 self._test_complete_flow( 1743 n_classes=n_classes, 1744 train_input_fn=train_input_fn, 1745 eval_input_fn=eval_input_fn, 1746 predict_input_fn=predict_input_fn, 1747 input_dimension=input_dimension, 1748 prediction_length=prediction_length) 1749 1750 def test_binary_classes_pandas_input_fn(self): 1751 self._test_pandas_input_fn(n_classes=2) 1752 1753 def test_multi_classes_pandas_input_fn(self): 1754 self._test_pandas_input_fn(n_classes=4) 1755 1756 def _test_input_fn_from_parse_example(self, n_classes): 1757 """Tests complete flow with input_fn constructed from parse_example.""" 1758 input_dimension = 2 1759 batch_size = 10 1760 prediction_length = batch_size 1761 data = np.linspace(0., 2., batch_size * input_dimension, dtype=np.float32) 1762 data = data.reshape(batch_size, input_dimension) 1763 target = np.array([1] * batch_size, dtype=np.int64) 1764 1765 serialized_examples = [] 1766 for x, y in zip(data, target): 1767 example = example_pb2.Example(features=feature_pb2.Features( 1768 feature={ 1769 'x': 1770 feature_pb2.Feature(float_list=feature_pb2.FloatList( 1771 value=x)), 1772 'y': 1773 feature_pb2.Feature(int64_list=feature_pb2.Int64List( 1774 value=[y])), 1775 })) 1776 serialized_examples.append(example.SerializeToString()) 1777 1778 feature_spec = { 1779 'x': parsing_ops.FixedLenFeature([input_dimension], dtypes.float32), 1780 'y': parsing_ops.FixedLenFeature([1], dtypes.int64), 1781 } 1782 1783 def _train_input_fn(): 1784 feature_map = parsing_ops.parse_example(serialized_examples, feature_spec) 1785 features = queue_parsed_features(feature_map) 1786 labels = features.pop('y') 1787 return features, labels 1788 1789 def _eval_input_fn(): 1790 feature_map = parsing_ops.parse_example( 1791 input_lib.limit_epochs(serialized_examples, num_epochs=1), 1792 feature_spec) 1793 features = queue_parsed_features(feature_map) 1794 labels = features.pop('y') 1795 return features, labels 1796 1797 def _predict_input_fn(): 1798 feature_map = parsing_ops.parse_example( 1799 input_lib.limit_epochs(serialized_examples, num_epochs=1), 1800 feature_spec) 1801 features = queue_parsed_features(feature_map) 1802 features.pop('y') 1803 return features, None 1804 1805 self._test_complete_flow( 1806 n_classes=n_classes, 1807 train_input_fn=_train_input_fn, 1808 eval_input_fn=_eval_input_fn, 1809 predict_input_fn=_predict_input_fn, 1810 input_dimension=input_dimension, 1811 prediction_length=prediction_length) 1812 1813 def test_binary_classes_input_fn_from_parse_example(self): 1814 self._test_input_fn_from_parse_example(n_classes=2) 1815 1816 def test_multi_classes_input_fn_from_parse_example(self): 1817 self._test_input_fn_from_parse_example(n_classes=4) 1818 1819 1820 class BaseLinearLogitFnTest(object): 1821 1822 def test_basic_logit_correctness(self): 1823 """linear_logit_fn simply wraps feature_column_lib.linear_model.""" 1824 age = feature_column_lib.numeric_column('age') 1825 with ops.Graph().as_default(): 1826 logit_fn = linear._linear_logit_fn_builder(units=2, feature_columns=[age]) 1827 logits = logit_fn(features={'age': [[23.], [31.]]}) 1828 bias_var = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, 1829 'linear_model/bias_weights')[0] 1830 age_var = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, 1831 'linear_model/age')[0] 1832 with tf_session.Session() as sess: 1833 sess.run([variables_lib.global_variables_initializer()]) 1834 self.assertAllClose([[0., 0.], [0., 0.]], logits.eval()) 1835 sess.run(bias_var.assign([10., 5.])) 1836 self.assertAllClose([[10., 5.], [10., 5.]], logits.eval()) 1837 sess.run(age_var.assign([[2.0, 3.0]])) 1838 # [2 * 23 + 10, 3 * 23 + 5] = [56, 74]. 1839 # [2 * 31 + 10, 3 * 31 + 5] = [72, 98] 1840 self.assertAllClose([[56., 74.], [72., 98.]], logits.eval()) 1841 1842 def test_compute_fraction_of_zero(self): 1843 """Tests the calculation of sparsity.""" 1844 age = feature_column_lib.numeric_column('age') 1845 occupation = feature_column_lib.categorical_column_with_hash_bucket( 1846 'occupation', hash_bucket_size=5) 1847 with ops.Graph().as_default(): 1848 cols_to_vars = {} 1849 feature_column_lib.linear_model( 1850 features={ 1851 'age': [[23.], [31.]], 1852 'occupation': [['doctor'], ['engineer']] 1853 }, 1854 feature_columns=[age, occupation], 1855 units=3, 1856 cols_to_vars=cols_to_vars) 1857 cols_to_vars.pop('bias') 1858 fraction_zero = linear._compute_fraction_of_zero(cols_to_vars) 1859 age_var = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, 1860 'linear_model/age')[0] 1861 with tf_session.Session() as sess: 1862 sess.run([variables_lib.global_variables_initializer()]) 1863 # Upon initialization, all variables will be zero. 1864 self.assertAllClose(1, fraction_zero.eval()) 1865 1866 sess.run(age_var.assign([[2.0, 0.0, -1.0]])) 1867 # 1 of the 3 age weights are zero, and all of the 15 (5 hash buckets 1868 # x 3-dim output) are zero. 1869 self.assertAllClose(16. / 18., fraction_zero.eval()) 1870 1871 1872 class BaseLinearWarmStartingTest(object): 1873 1874 def __init__(self, _linear_classifier_fn, _linear_regressor_fn): 1875 self._linear_classifier_fn = _linear_classifier_fn 1876 self._linear_regressor_fn = _linear_regressor_fn 1877 1878 def setUp(self): 1879 # Create a directory to save our old checkpoint and vocabularies to. 1880 self._ckpt_and_vocab_dir = tempfile.mkdtemp() 1881 1882 # Make a dummy input_fn. 1883 def _input_fn(): 1884 features = { 1885 'age': [[23.], [31.]], 1886 'age_in_years': [[23.], [31.]], 1887 'occupation': [['doctor'], ['consultant']] 1888 } 1889 return features, [0, 1] 1890 1891 self._input_fn = _input_fn 1892 1893 def tearDown(self): 1894 # Clean up checkpoint / vocab dir. 1895 writer_cache.FileWriterCache.clear() 1896 shutil.rmtree(self._ckpt_and_vocab_dir) 1897 1898 def test_classifier_basic_warm_starting(self): 1899 """Tests correctness of LinearClassifier default warm-start.""" 1900 age = feature_column_lib.numeric_column('age') 1901 1902 # Create a LinearClassifier and train to save a checkpoint. 1903 linear_classifier = self._linear_classifier_fn( 1904 feature_columns=[age], 1905 model_dir=self._ckpt_and_vocab_dir, 1906 n_classes=4, 1907 optimizer='SGD') 1908 linear_classifier.train(input_fn=self._input_fn, max_steps=1) 1909 1910 # Create a second LinearClassifier, warm-started from the first. Use a 1911 # learning_rate = 0.0 optimizer to check values (use SGD so we don't have 1912 # accumulator values that change). 1913 warm_started_linear_classifier = self._linear_classifier_fn( 1914 feature_columns=[age], 1915 n_classes=4, 1916 optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.0), 1917 warm_start_from=linear_classifier.model_dir) 1918 1919 warm_started_linear_classifier.train(input_fn=self._input_fn, max_steps=1) 1920 for variable_name in warm_started_linear_classifier.get_variable_names(): 1921 self.assertAllClose( 1922 linear_classifier.get_variable_value(variable_name), 1923 warm_started_linear_classifier.get_variable_value(variable_name)) 1924 1925 def test_regressor_basic_warm_starting(self): 1926 """Tests correctness of LinearRegressor default warm-start.""" 1927 age = feature_column_lib.numeric_column('age') 1928 1929 # Create a LinearRegressor and train to save a checkpoint. 1930 linear_regressor = self._linear_regressor_fn( 1931 feature_columns=[age], 1932 model_dir=self._ckpt_and_vocab_dir, 1933 optimizer='SGD') 1934 linear_regressor.train(input_fn=self._input_fn, max_steps=1) 1935 1936 # Create a second LinearRegressor, warm-started from the first. Use a 1937 # learning_rate = 0.0 optimizer to check values (use SGD so we don't have 1938 # accumulator values that change). 1939 warm_started_linear_regressor = self._linear_regressor_fn( 1940 feature_columns=[age], 1941 optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.0), 1942 warm_start_from=linear_regressor.model_dir) 1943 1944 warm_started_linear_regressor.train(input_fn=self._input_fn, max_steps=1) 1945 for variable_name in warm_started_linear_regressor.get_variable_names(): 1946 self.assertAllClose( 1947 linear_regressor.get_variable_value(variable_name), 1948 warm_started_linear_regressor.get_variable_value(variable_name)) 1949 1950 def test_warm_starting_selective_variables(self): 1951 """Tests selecting variables to warm-start.""" 1952 age = feature_column_lib.numeric_column('age') 1953 1954 # Create a LinearClassifier and train to save a checkpoint. 1955 linear_classifier = self._linear_classifier_fn( 1956 feature_columns=[age], 1957 model_dir=self._ckpt_and_vocab_dir, 1958 n_classes=4, 1959 optimizer='SGD') 1960 linear_classifier.train(input_fn=self._input_fn, max_steps=1) 1961 1962 # Create a second LinearClassifier, warm-started from the first. Use a 1963 # learning_rate = 0.0 optimizer to check values (use SGD so we don't have 1964 # accumulator values that change). 1965 warm_started_linear_classifier = self._linear_classifier_fn( 1966 feature_columns=[age], 1967 n_classes=4, 1968 optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.0), 1969 # The provided regular expression will only warm-start the age variable 1970 # and not the bias. 1971 warm_start_from=warm_starting_util.WarmStartSettings( 1972 ckpt_to_initialize_from=linear_classifier.model_dir, 1973 vars_to_warm_start='.*(age).*')) 1974 1975 warm_started_linear_classifier.train(input_fn=self._input_fn, max_steps=1) 1976 self.assertAllClose( 1977 linear_classifier.get_variable_value(AGE_WEIGHT_NAME), 1978 warm_started_linear_classifier.get_variable_value(AGE_WEIGHT_NAME)) 1979 # Bias should still be zero from initialization. 1980 self.assertAllClose( 1981 [0.0] * 4, warm_started_linear_classifier.get_variable_value(BIAS_NAME)) 1982 1983 def test_warm_starting_with_vocab_remapping_and_partitioning(self): 1984 """Tests warm-starting with vocab remapping and partitioning.""" 1985 vocab_list = ['doctor', 'lawyer', 'consultant'] 1986 vocab_file = os.path.join(self._ckpt_and_vocab_dir, 'occupation_vocab') 1987 with open(vocab_file, 'w') as f: 1988 f.write('\n'.join(vocab_list)) 1989 occupation = feature_column_lib.categorical_column_with_vocabulary_file( 1990 'occupation', 1991 vocabulary_file=vocab_file, 1992 vocabulary_size=len(vocab_list)) 1993 1994 # Create a LinearClassifier and train to save a checkpoint. 1995 partitioner = partitioned_variables.fixed_size_partitioner(num_shards=2) 1996 linear_classifier = self._linear_classifier_fn( 1997 feature_columns=[occupation], 1998 model_dir=self._ckpt_and_vocab_dir, 1999 n_classes=4, 2000 optimizer='SGD', 2001 partitioner=partitioner) 2002 linear_classifier.train(input_fn=self._input_fn, max_steps=1) 2003 2004 # Create a second LinearClassifier, warm-started from the first. Use a 2005 # learning_rate = 0.0 optimizer to check values (use SGD so we don't have 2006 # accumulator values that change). Use a new FeatureColumn with a 2007 # different vocabulary for occupation. 2008 new_vocab_list = ['doctor', 'consultant', 'engineer'] 2009 new_vocab_file = os.path.join(self._ckpt_and_vocab_dir, 2010 'new_occupation_vocab') 2011 with open(new_vocab_file, 'w') as f: 2012 f.write('\n'.join(new_vocab_list)) 2013 new_occupation = feature_column_lib.categorical_column_with_vocabulary_file( 2014 'occupation', 2015 vocabulary_file=new_vocab_file, 2016 vocabulary_size=len(new_vocab_list)) 2017 # We can create our VocabInfo object from the new and old occupation 2018 # FeatureColumn's. 2019 occupation_vocab_info = warm_starting_util.VocabInfo( 2020 new_vocab=new_occupation.vocabulary_file, 2021 new_vocab_size=new_occupation.vocabulary_size, 2022 num_oov_buckets=new_occupation.num_oov_buckets, 2023 old_vocab=occupation.vocabulary_file, 2024 old_vocab_size=occupation.vocabulary_size, 2025 # Can't use constant_initializer with load_and_remap. In practice, 2026 # use a truncated normal initializer. 2027 backup_initializer=init_ops.random_uniform_initializer( 2028 minval=0.39, maxval=0.39)) 2029 warm_started_linear_classifier = self._linear_classifier_fn( 2030 feature_columns=[occupation], 2031 n_classes=4, 2032 optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.0), 2033 warm_start_from=warm_starting_util.WarmStartSettings( 2034 ckpt_to_initialize_from=linear_classifier.model_dir, 2035 var_name_to_vocab_info={ 2036 OCCUPATION_WEIGHT_NAME: occupation_vocab_info 2037 }, 2038 # Explicitly providing None here will only warm-start variables 2039 # referenced in var_name_to_vocab_info (the bias will not be 2040 # warm-started). 2041 vars_to_warm_start=None), 2042 partitioner=partitioner) 2043 2044 warm_started_linear_classifier.train(input_fn=self._input_fn, max_steps=1) 2045 # 'doctor' was ID-0 and still ID-0. 2046 self.assertAllClose( 2047 linear_classifier.get_variable_value(OCCUPATION_WEIGHT_NAME)[0, :], 2048 warm_started_linear_classifier.get_variable_value( 2049 OCCUPATION_WEIGHT_NAME)[0, :]) 2050 # 'consultant' was ID-2 and now ID-1. 2051 self.assertAllClose( 2052 linear_classifier.get_variable_value(OCCUPATION_WEIGHT_NAME)[2, :], 2053 warm_started_linear_classifier.get_variable_value( 2054 OCCUPATION_WEIGHT_NAME)[1, :]) 2055 # 'engineer' is a new entry and should be initialized with the 2056 # backup_initializer in VocabInfo. 2057 self.assertAllClose([0.39] * 4, 2058 warm_started_linear_classifier.get_variable_value( 2059 OCCUPATION_WEIGHT_NAME)[2, :]) 2060 # Bias should still be zero (from initialization logic). 2061 self.assertAllClose( 2062 [0.0] * 4, warm_started_linear_classifier.get_variable_value(BIAS_NAME)) 2063 2064 def test_warm_starting_with_naming_change(self): 2065 """Tests warm-starting with a Tensor name remapping.""" 2066 age_in_years = feature_column_lib.numeric_column('age_in_years') 2067 2068 # Create a LinearClassifier and train to save a checkpoint. 2069 linear_classifier = self._linear_classifier_fn( 2070 feature_columns=[age_in_years], 2071 model_dir=self._ckpt_and_vocab_dir, 2072 n_classes=4, 2073 optimizer='SGD') 2074 linear_classifier.train(input_fn=self._input_fn, max_steps=1) 2075 2076 # Create a second LinearClassifier, warm-started from the first. Use a 2077 # learning_rate = 0.0 optimizer to check values (use SGD so we don't have 2078 # accumulator values that change). 2079 warm_started_linear_classifier = self._linear_classifier_fn( 2080 feature_columns=[feature_column_lib.numeric_column('age')], 2081 n_classes=4, 2082 optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.0), 2083 # The 'age' variable correspond to the 'age_in_years' variable in the 2084 # previous model. 2085 warm_start_from=warm_starting_util.WarmStartSettings( 2086 ckpt_to_initialize_from=linear_classifier.model_dir, 2087 var_name_to_prev_var_name={ 2088 AGE_WEIGHT_NAME: AGE_WEIGHT_NAME.replace('age', 'age_in_years') 2089 })) 2090 2091 warm_started_linear_classifier.train(input_fn=self._input_fn, max_steps=1) 2092 self.assertAllClose( 2093 linear_classifier.get_variable_value( 2094 AGE_WEIGHT_NAME.replace('age', 'age_in_years')), 2095 warm_started_linear_classifier.get_variable_value(AGE_WEIGHT_NAME)) 2096 # The bias is also warm-started (with no name remapping). 2097 self.assertAllClose( 2098 linear_classifier.get_variable_value(BIAS_NAME), 2099 warm_started_linear_classifier.get_variable_value(BIAS_NAME)) 2100
