Skip to content

Commit b4af21d

Browse files
autoihautoih
authored
Removed tests out of run_in_graph_and_eager_mode in conditional gradient (#1610)
1 parent 8ce8d09 commit b4af21d

File tree

1 file changed

+134
-241
lines changed

1 file changed

+134
-241
lines changed

‎tensorflow_addons/optimizers/tests/conditional_gradient_test.py‎

Lines changed: 134 additions & 241 deletions
Original file line numberDiff line numberDiff line change
@@ -942,251 +942,144 @@ def test_sparse_frobenius():
942942
)
943943

944944

945-
@test_utils.run_all_in_graph_and_eager_modes
946-
class ConditionalGradientTest(tf.test.TestCase):
947-
def testSharingFrobenius(self):
948-
for dtype in [tf.half, tf.float32, tf.float64]:
949-
with self.cached_session():
950-
var0 = tf.Variable([1.0, 2.0], dtype=dtype)
951-
var1 = tf.Variable([3.0, 4.0], dtype=dtype)
952-
grads0 = tf.constant([0.1, 0.1], dtype=dtype)
953-
grads1 = tf.constant([0.01, 0.01], dtype=dtype)
954-
norm0 = tf.math.reduce_sum(grads0 ** 2) ** 0.5
955-
norm1 = tf.math.reduce_sum(grads1 ** 2) ** 0.5
956-
learning_rate = 0.1
957-
lambda_ = 0.1
958-
ord = "fro"
959-
cg_opt = cg_lib.ConditionalGradient(
960-
learning_rate=learning_rate, lambda_=lambda_, ord=ord
961-
)
962-
cg_update1 = cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
963-
cg_update2 = cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
964-
if not tf.executing_eagerly():
965-
self.evaluate(tf.compat.v1.global_variables_initializer())
966-
# Fetch params to validate initial values
967-
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
968-
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
969-
970-
# Check we have slots
971-
self.assertEqual(["conditional_gradient"], cg_opt.get_slot_names())
972-
slot0 = cg_opt.get_slot(var0, "conditional_gradient")
973-
self.assertEquals(slot0.get_shape(), var0.get_shape())
974-
slot1 = cg_opt.get_slot(var1, "conditional_gradient")
975-
self.assertEquals(slot1.get_shape(), var1.get_shape())
976-
977-
if not tf.executing_eagerly():
978-
self.assertFalse(slot0 in tf.compat.v1.trainable_variables())
979-
self.assertFalse(slot1 in tf.compat.v1.trainable_variables())
980-
# Because in the eager mode, as we declare two cg_update
981-
# variables, it already altomatically finish executing them.
982-
# Thus, we cannot test the param value at this time for
983-
# eager mode. We can only test the final value of param
984-
# after the second execution.
985-
if not tf.executing_eagerly():
986-
self.evaluate(cg_update1)
987-
# Check that the parameters have been updated.
988-
norm0 = self.evaluate(norm0)
989-
norm1 = self.evaluate(norm1)
990-
self.assertAllCloseAccordingToType(
991-
np.array(
992-
[
993-
1.0 * learning_rate
994-
- (1 - learning_rate) * lambda_ * 0.1 / norm0,
995-
2.0 * learning_rate
996-
- (1 - learning_rate) * lambda_ * 0.1 / norm0,
997-
]
998-
),
999-
self.evaluate(var0),
945+
@pytest.mark.usefixtures("maybe_run_functions_eagerly")
946+
@pytest.mark.parametrize("dtype", [tf.half, tf.float32, tf.float64])
947+
def test_sharing_frobenius(dtype):
948+
var0 = tf.Variable([1.0, 2.0], dtype=dtype)
949+
var1 = tf.Variable([3.0, 4.0], dtype=dtype)
950+
grads0 = tf.constant([0.1, 0.1], dtype=dtype)
951+
grads1 = tf.constant([0.01, 0.01], dtype=dtype)
952+
norm0 = tf.math.reduce_sum(grads0 ** 2) ** 0.5
953+
norm1 = tf.math.reduce_sum(grads1 ** 2) ** 0.5
954+
learning_rate = 0.1
955+
lambda_ = 0.1
956+
ord = "fro"
957+
cg_opt = cg_lib.ConditionalGradient(
958+
learning_rate=learning_rate, lambda_=lambda_, ord=ord
959+
)
960+
_ = cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
961+
962+
# Check we have slots
963+
assert ["conditional_gradient"] == cg_opt.get_slot_names()
964+
slot0 = cg_opt.get_slot(var0, "conditional_gradient")
965+
assert slot0.get_shape() == var0.get_shape()
966+
slot1 = cg_opt.get_slot(var1, "conditional_gradient")
967+
assert slot1.get_shape() == var1.get_shape()
968+
969+
# Because in the eager mode, as we declare two cg_update
970+
# variables, it already altomatically finish executing them.
971+
# Thus, we cannot test the param value at this time for
972+
# eager mode. We can only test the final value of param
973+
# after the second execution.
974+
975+
# Step 2: the second conditional_gradient contain
976+
# the previous update.
977+
# Check that the parameters have been updated.
978+
cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
979+
test_utils.assert_allclose_according_to_type(
980+
np.array(
981+
[
982+
(1.0 * learning_rate - (1 - learning_rate) * lambda_ * 0.1 / norm0)
983+
* learning_rate
984+
- (1 - learning_rate) * lambda_ * 0.1 / norm0,
985+
(2.0 * learning_rate - (1 - learning_rate) * lambda_ * 0.1 / norm0)
986+
* learning_rate
987+
- (1 - learning_rate) * lambda_ * 0.1 / norm0,
988+
]
989+
),
990+
var0.numpy(),
991+
)
992+
test_utils.assert_allclose_according_to_type(
993+
np.array(
994+
[
995+
(3.0 * learning_rate - (1 - learning_rate) * lambda_ * 0.01 / norm1)
996+
* learning_rate
997+
- (1 - learning_rate) * lambda_ * 0.01 / norm1,
998+
(4.0 * learning_rate - (1 - learning_rate) * lambda_ * 0.01 / norm1)
999+
* learning_rate
1000+
- (1 - learning_rate) * lambda_ * 0.01 / norm1,
1001+
]
1002+
),
1003+
var1.numpy(),
1004+
)
1005+
1006+
1007+
@pytest.mark.usefixtures("maybe_run_functions_eagerly")
1008+
def test_sharing_nuclear():
1009+
# TODO:
1010+
# To address the issue #36764.
1011+
for dtype in _dtypes_with_checking_system(
1012+
use_gpu=tf.test.is_gpu_available(), system=platform.system()
1013+
):
1014+
var0 = tf.Variable([1.0, 2.0], dtype=dtype)
1015+
var1 = tf.Variable([3.0, 4.0], dtype=dtype)
1016+
grads0 = tf.constant([0.1, 0.1], dtype=dtype)
1017+
grads1 = tf.constant([0.01, 0.01], dtype=dtype)
1018+
top_singular_vector0 = cg_lib.ConditionalGradient._top_singular_vector(grads0)
1019+
top_singular_vector1 = cg_lib.ConditionalGradient._top_singular_vector(grads1)
1020+
learning_rate = 0.1
1021+
lambda_ = 0.1
1022+
ord = "nuclear"
1023+
cg_opt = cg_lib.ConditionalGradient(
1024+
learning_rate=learning_rate, lambda_=lambda_, ord=ord
1025+
)
1026+
_ = cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
1027+
1028+
# Check we have slots
1029+
assert ["conditional_gradient"] == cg_opt.get_slot_names()
1030+
slot0 = cg_opt.get_slot(var0, "conditional_gradient")
1031+
assert slot0.get_shape() == var0.get_shape()
1032+
slot1 = cg_opt.get_slot(var1, "conditional_gradient")
1033+
assert slot1.get_shape() == var1.get_shape()
1034+
1035+
# Because in the eager mode, as we declare two cg_update
1036+
# variables, it already altomatically finish executing them.
1037+
# Thus, we cannot test the param value at this time for
1038+
# eager mode. We can only test the final value of param
1039+
# after the second execution.
1040+
1041+
# Step 2: the second conditional_gradient contain
1042+
# the previous update.
1043+
# Check that the parameters have been updated.
1044+
cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
1045+
test_utils.assert_allclose_according_to_type(
1046+
np.array(
1047+
[
1048+
(
1049+
1.0 * learning_rate
1050+
- (1 - learning_rate) * lambda_ * top_singular_vector0[0]
10001051
)
1001-
self.assertAllCloseAccordingToType(
1002-
np.array(
1003-
[
1004-
3.0 * learning_rate
1005-
- (1 - learning_rate) * lambda_ * 0.01 / norm1,
1006-
4.0 * learning_rate
1007-
- (1 - learning_rate) * lambda_ * 0.01 / norm1,
1008-
]
1009-
),
1010-
self.evaluate(var1),
1052+
* learning_rate
1053+
- (1 - learning_rate) * lambda_ * top_singular_vector0[0],
1054+
(
1055+
2.0 * learning_rate
1056+
- (1 - learning_rate) * lambda_ * top_singular_vector0[1]
10111057
)
1012-
1013-
# Step 2: the second conditional_gradient contain
1014-
# the previous update.
1015-
if not tf.executing_eagerly():
1016-
self.evaluate(cg_update2)
1017-
# Check that the parameters have been updated.
1018-
self.assertAllCloseAccordingToType(
1019-
np.array(
1020-
[
1021-
(
1022-
1.0 * learning_rate
1023-
- (1 - learning_rate) * lambda_ * 0.1 / norm0
1024-
)
1025-
* learning_rate
1026-
- (1 - learning_rate) * lambda_ * 0.1 / norm0,
1027-
(
1028-
2.0 * learning_rate
1029-
- (1 - learning_rate) * lambda_ * 0.1 / norm0
1030-
)
1031-
* learning_rate
1032-
- (1 - learning_rate) * lambda_ * 0.1 / norm0,
1033-
]
1034-
),
1035-
self.evaluate(var0),
1036-
)
1037-
self.assertAllCloseAccordingToType(
1038-
np.array(
1039-
[
1040-
(
1041-
3.0 * learning_rate
1042-
- (1 - learning_rate) * lambda_ * 0.01 / norm1
1043-
)
1044-
* learning_rate
1045-
- (1 - learning_rate) * lambda_ * 0.01 / norm1,
1046-
(
1047-
4.0 * learning_rate
1048-
- (1 - learning_rate) * lambda_ * 0.01 / norm1
1049-
)
1050-
* learning_rate
1051-
- (1 - learning_rate) * lambda_ * 0.01 / norm1,
1052-
]
1053-
),
1054-
self.evaluate(var1),
1055-
)
1056-
1057-
def testSharingNuclear(self):
1058-
# TODO:
1059-
# To address the issue #36764.
1060-
for dtype in _dtypes_with_checking_system(
1061-
use_gpu=tf.test.is_gpu_available(), system=platform.system()
1062-
):
1063-
with self.cached_session():
1064-
var0 = tf.Variable([1.0, 2.0], dtype=dtype)
1065-
var1 = tf.Variable([3.0, 4.0], dtype=dtype)
1066-
grads0 = tf.constant([0.1, 0.1], dtype=dtype)
1067-
grads1 = tf.constant([0.01, 0.01], dtype=dtype)
1068-
top_singular_vector0 = cg_lib.ConditionalGradient._top_singular_vector(
1069-
grads0
1070-
)
1071-
top_singular_vector1 = cg_lib.ConditionalGradient._top_singular_vector(
1072-
grads1
1073-
)
1074-
learning_rate = 0.1
1075-
lambda_ = 0.1
1076-
ord = "nuclear"
1077-
cg_opt = cg_lib.ConditionalGradient(
1078-
learning_rate=learning_rate, lambda_=lambda_, ord=ord
1079-
)
1080-
cg_update1 = cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
1081-
cg_update2 = cg_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
1082-
if not tf.executing_eagerly():
1083-
self.evaluate(tf.compat.v1.global_variables_initializer())
1084-
# Fetch params to validate initial values
1085-
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
1086-
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
1087-
1088-
# Check we have slots
1089-
self.assertEqual(["conditional_gradient"], cg_opt.get_slot_names())
1090-
slot0 = cg_opt.get_slot(var0, "conditional_gradient")
1091-
self.assertEquals(slot0.get_shape(), var0.get_shape())
1092-
slot1 = cg_opt.get_slot(var1, "conditional_gradient")
1093-
self.assertEquals(slot1.get_shape(), var1.get_shape())
1094-
1095-
if not tf.executing_eagerly():
1096-
self.assertFalse(slot0 in tf.compat.v1.trainable_variables())
1097-
self.assertFalse(slot1 in tf.compat.v1.trainable_variables())
1098-
# Because in the eager mode, as we declare two cg_update
1099-
# variables, it already altomatically finish executing them.
1100-
# Thus, we cannot test the param value at this time for
1101-
# eager mode. We can only test the final value of param
1102-
# after the second execution.
1103-
if not tf.executing_eagerly():
1104-
self.evaluate(cg_update1)
1105-
# Check that the parameters have been updated.
1106-
top_singular_vector0 = self.evaluate(top_singular_vector0)
1107-
top_singular_vector1 = self.evaluate(top_singular_vector1)
1108-
self.assertAllCloseAccordingToType(
1109-
np.array(
1110-
[
1111-
1.0 * learning_rate
1112-
- (1 - learning_rate)
1113-
* lambda_
1114-
* top_singular_vector0[0],
1115-
2.0 * learning_rate
1116-
- (1 - learning_rate)
1117-
* lambda_
1118-
* top_singular_vector0[1],
1119-
]
1120-
),
1121-
self.evaluate(var0),
1058+
* learning_rate
1059+
- (1 - learning_rate) * lambda_ * top_singular_vector0[1],
1060+
]
1061+
),
1062+
var0.numpy(),
1063+
)
1064+
test_utils.assert_allclose_according_to_type(
1065+
np.array(
1066+
[
1067+
(
1068+
3.0 * learning_rate
1069+
- (1 - learning_rate) * lambda_ * top_singular_vector1[0]
11221070
)
1123-
self.assertAllCloseAccordingToType(
1124-
np.array(
1125-
[
1126-
3.0 * learning_rate
1127-
- (1 - learning_rate)
1128-
* lambda_
1129-
* top_singular_vector1[0],
1130-
4.0 * learning_rate
1131-
- (1 - learning_rate)
1132-
* lambda_
1133-
* top_singular_vector1[1],
1134-
]
1135-
),
1136-
self.evaluate(var1),
1071+
* learning_rate
1072+
- (1 - learning_rate) * lambda_ * top_singular_vector1[0],
1073+
(
1074+
4.0 * learning_rate
1075+
- (1 - learning_rate) * lambda_ * top_singular_vector1[1]
11371076
)
1138-
1139-
# Step 2: the second conditional_gradient contain
1140-
# the previous update.
1141-
if not tf.executing_eagerly():
1142-
self.evaluate(cg_update2)
1143-
# Check that the parameters have been updated.
1144-
self.assertAllCloseAccordingToType(
1145-
np.array(
1146-
[
1147-
(
1148-
1.0 * learning_rate
1149-
- (1 - learning_rate)
1150-
* lambda_
1151-
* top_singular_vector0[0]
1152-
)
1153-
* learning_rate
1154-
- (1 - learning_rate) * lambda_ * top_singular_vector0[0],
1155-
(
1156-
2.0 * learning_rate
1157-
- (1 - learning_rate)
1158-
* lambda_
1159-
* top_singular_vector0[1]
1160-
)
1161-
* learning_rate
1162-
- (1 - learning_rate) * lambda_ * top_singular_vector0[1],
1163-
]
1164-
),
1165-
self.evaluate(var0),
1166-
)
1167-
self.assertAllCloseAccordingToType(
1168-
np.array(
1169-
[
1170-
(
1171-
3.0 * learning_rate
1172-
- (1 - learning_rate)
1173-
* lambda_
1174-
* top_singular_vector1[0]
1175-
)
1176-
* learning_rate
1177-
- (1 - learning_rate) * lambda_ * top_singular_vector1[0],
1178-
(
1179-
4.0 * learning_rate
1180-
- (1 - learning_rate)
1181-
* lambda_
1182-
* top_singular_vector1[1]
1183-
)
1184-
* learning_rate
1185-
- (1 - learning_rate) * lambda_ * top_singular_vector1[1],
1186-
]
1187-
),
1188-
self.evaluate(var1),
1189-
)
1077+
* learning_rate
1078+
- (1 - learning_rate) * lambda_ * top_singular_vector1[1],
1079+
]
1080+
),
1081+
var1.numpy(),
1082+
)
11901083

11911084

11921085
def _db_params_nuclear_cg01():

0 commit comments

Comments
 (0)