From 22ed3afbdaebcd69bdb01f9cfe7dbb2d088c6df6 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Thu, 31 Mar 2022 15:13:19 +0530 Subject: [PATCH 1/9] adding alternative option in local_moran and moral_local_rate --- esda/moran.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/esda/moran.py b/esda/moran.py index 5bc85316..109ff073 100644 --- a/esda/moran.py +++ b/esda/moran.py @@ -889,6 +889,9 @@ class Moran_Local(object): value to use as a weight for the "fake" neighbor for every island. If numpy.nan, will propagate to the final local statistic depending on the `stat_func`. If 0, then the lag is always zero for islands. + alternative: string + possible values -> "one-tailed"/ "two-tailed" + default value ->"two-tailed" Attributes ---------- @@ -1005,6 +1008,7 @@ def __init__( keep_simulations=True, seed=None, island_weight=0, + alternative = "two-tailed" ): y = np.asarray(y).flatten() self.y = y @@ -1063,6 +1067,12 @@ def __init__( self.VI_sim = np.nan self.z_sim = np.nan self.p_z_sim = np.nan + if alternative=="one-tailed": + self.p_z_sim= self.p_z_sim*2 + folded_replicates = np.abs(self.rlisas - np.median(self.rlisas, axis=1, keepdims=True)) + self.p_sim = (folded_replicates >= np.abs(self.Is[:,None])).mean(axis=1) + + def __calc(self, w, z): zl = slag(w, z) @@ -1575,6 +1585,7 @@ def __init__( keep_simulations=True, seed=None, island_weight=0, + alternative="two-tailed" ): e = np.asarray(e).flatten() b = np.asarray(b).flatten() @@ -1592,6 +1603,7 @@ def __init__( n_jobs=n_jobs, keep_simulations=keep_simulations, seed=seed, + alternative=alternative ) @classmethod From cdcd839a1208ffb04f092efc357068b872f9a06d Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Thu, 31 Mar 2022 17:58:56 +0530 Subject: [PATCH 2/9] convert Moran_BV_matrix into a class --- esda/moran.py | 51 +++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/esda/moran.py b/esda/moran.py index 109ff073..82963ae0 100644 --- a/esda/moran.py +++ b/esda/moran.py @@ -506,9 +506,9 @@ def by_col( ) -def Moran_BV_matrix(variables, w, permutations=0, varnames=None): +class Moran_BV_Matrix: """ - Bivariate Moran Matrix + Bivariate Moran Matrix Class Calculates bivariate Moran between all pairs of a set of variables. @@ -531,7 +531,6 @@ def Moran_BV_matrix(variables, w, permutations=0, varnames=None): results : dictionary (i, j) is the key for the pair of variables, values are the Moran_BV objects. - Examples -------- @@ -556,32 +555,36 @@ def Moran_BV_matrix(variables, w, permutations=0, varnames=None): check values - >>> round(res[(0, 1)].I,7) + >>> round(res.matrix[(0, 1)].I,7) 0.1936261 - >>> round(res[(3, 0)].I,7) + >>> round(res.matrix[(3, 0)].I,7) 0.3770138 """ - try: - # check if pandas is installed - import pandas - - if isinstance(variables, pandas.DataFrame): - # if yes use variables as df and convert to numpy_array - varnames = pandas.Index.tolist(variables.columns) - variables_n = [] - for var in varnames: - variables_n.append(variables[str(var)].values) - else: + def __init__(self,variables, w, permutations=0, varnames=None): + self.variables = variables + self.w = w + self.permutations = permutations + self.varnames = varnames + try: + # check if pandas is installed + import pandas + + if isinstance(variables, pandas.DataFrame): + # if yes use variables as df and convert to numpy_array + varnames = pandas.Index.tolist(variables.columns) + variables_n = [] + for var in varnames: + variables_n.append(variables[str(var)].values) + else: + variables_n = variables + except ImportError: variables_n = variables - except ImportError: - variables_n = variables - - results = _Moran_BV_Matrix_array( - variables=variables_n, w=w, permutations=permutations, varnames=varnames - ) - return results - + + self.matrix = _Moran_BV_Matrix_array( + variables=variables_n, w=self.w, permutations=self.permutations, varnames=self.varnames + ) + def _Moran_BV_Matrix_array(variables, w, permutations=0, varnames=None): """ From f197358902af8ce0ac1cfa07d736ad3233524c14 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Fri, 1 Apr 2022 01:01:04 +0530 Subject: [PATCH 3/9] convert Moran_BV_matrix into a class correct --- esda/moran.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esda/moran.py b/esda/moran.py index 82963ae0..cb7e639d 100644 --- a/esda/moran.py +++ b/esda/moran.py @@ -506,7 +506,7 @@ def by_col( ) -class Moran_BV_Matrix: +class Moran_BV_matrix: """ Bivariate Moran Matrix Class From c4892d973a1db96a734e9db68c8ba4c8ab854773 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Thu, 7 Apr 2022 10:47:01 +0530 Subject: [PATCH 4/9] changed test and documentation --- esda/tests/test_moran.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esda/tests/test_moran.py b/esda/tests/test_moran.py index 65319885..8cc06933 100644 --- a/esda/tests/test_moran.py +++ b/esda/tests/test_moran.py @@ -96,8 +96,8 @@ def setUp(self): def test_Moran_BV_matrix(self): res = moran.Moran_BV_matrix(self.vars, self.w, varnames=self.names) - self.assertAlmostEqual(res[(0, 1)].I, 0.19362610652874668) - self.assertAlmostEqual(res[(3, 0)].I, 0.37701382542927858) + self.assertAlmostEqual(res.matrix[(0, 1)].I, 0.19362610652874668) + self.assertAlmostEqual(res.matrix[(3, 0)].I, 0.37701382542927858) class Moran_Local_Tester(unittest.TestCase): From 5b4435ddd65d1996b22844d979d803052a0e8908 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Fri, 8 Apr 2022 16:03:02 +0530 Subject: [PATCH 5/9] reverting Moran_BV_matrix to default --- esda/moran.py | 63 +++++++++++++++------------------------- esda/tests/test_moran.py | 4 +-- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/esda/moran.py b/esda/moran.py index cb7e639d..73d96889 100644 --- a/esda/moran.py +++ b/esda/moran.py @@ -506,12 +506,10 @@ def by_col( ) -class Moran_BV_matrix: +def Moran_BV_matrix(variables, w, permutations=0, varnames=None): """ - Bivariate Moran Matrix Class - + Bivariate Moran Matrix Calculates bivariate Moran between all pairs of a set of variables. - Parameters ---------- variables : array or pandas.DataFrame @@ -533,58 +531,43 @@ class Moran_BV_matrix: the Moran_BV objects. Examples -------- - open dbf - >>> import libpysal >>> f = libpysal.io.open(libpysal.examples.get_path("sids2.dbf")) - pull of selected variables from dbf and create numpy arrays for each - >>> varnames = ['SIDR74', 'SIDR79', 'NWR74', 'NWR79'] >>> vars = [np.array(f.by_col[var]) for var in varnames] - create a contiguity matrix from an external gal file - >>> w = libpysal.io.open(libpysal.examples.get_path("sids2.gal")).read() - create an instance of Moran_BV_matrix - >>> from esda.moran import Moran_BV_matrix >>> res = Moran_BV_matrix(vars, w, varnames = varnames) - check values - - >>> round(res.matrix[(0, 1)].I,7) + >>> round(res[(0, 1)].I,7) 0.1936261 - >>> round(res.matrix[(3, 0)].I,7) + >>> round(res[(3, 0)].I,7) 0.3770138 - """ - def __init__(self,variables, w, permutations=0, varnames=None): - self.variables = variables - self.w = w - self.permutations = permutations - self.varnames = varnames - try: - # check if pandas is installed - import pandas - - if isinstance(variables, pandas.DataFrame): - # if yes use variables as df and convert to numpy_array - varnames = pandas.Index.tolist(variables.columns) - variables_n = [] - for var in varnames: - variables_n.append(variables[str(var)].values) - else: - variables_n = variables - except ImportError: + try: + # check if pandas is installed + import pandas + + if isinstance(variables, pandas.DataFrame): + # if yes use variables as df and convert to numpy_array + varnames = pandas.Index.tolist(variables.columns) + variables_n = [] + for var in varnames: + variables_n.append(variables[str(var)].values) + else: variables_n = variables - - self.matrix = _Moran_BV_Matrix_array( - variables=variables_n, w=self.w, permutations=self.permutations, varnames=self.varnames - ) - + except ImportError: + variables_n = variables + + results = _Moran_BV_Matrix_array( + variables=variables_n, w=w, permutations=permutations, varnames=varnames + ) + return results + def _Moran_BV_Matrix_array(variables, w, permutations=0, varnames=None): """ diff --git a/esda/tests/test_moran.py b/esda/tests/test_moran.py index 8cc06933..65319885 100644 --- a/esda/tests/test_moran.py +++ b/esda/tests/test_moran.py @@ -96,8 +96,8 @@ def setUp(self): def test_Moran_BV_matrix(self): res = moran.Moran_BV_matrix(self.vars, self.w, varnames=self.names) - self.assertAlmostEqual(res.matrix[(0, 1)].I, 0.19362610652874668) - self.assertAlmostEqual(res.matrix[(3, 0)].I, 0.37701382542927858) + self.assertAlmostEqual(res[(0, 1)].I, 0.19362610652874668) + self.assertAlmostEqual(res[(3, 0)].I, 0.37701382542927858) class Moran_Local_Tester(unittest.TestCase): From a89ae03fddb8ea702c9ce99e18ae6ebcce148190 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Tue, 12 Apr 2022 14:41:12 +0530 Subject: [PATCH 6/9] updated documentation , test coverage is still left --- esda/moran.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/esda/moran.py b/esda/moran.py index 73d96889..15f97a9e 100644 --- a/esda/moran.py +++ b/esda/moran.py @@ -509,7 +509,9 @@ def by_col( def Moran_BV_matrix(variables, w, permutations=0, varnames=None): """ Bivariate Moran Matrix + Calculates bivariate Moran between all pairs of a set of variables. + Parameters ---------- variables : array or pandas.DataFrame @@ -529,24 +531,36 @@ def Moran_BV_matrix(variables, w, permutations=0, varnames=None): results : dictionary (i, j) is the key for the pair of variables, values are the Moran_BV objects. + Examples -------- + open dbf + >>> import libpysal >>> f = libpysal.io.open(libpysal.examples.get_path("sids2.dbf")) + pull of selected variables from dbf and create numpy arrays for each + >>> varnames = ['SIDR74', 'SIDR79', 'NWR74', 'NWR79'] >>> vars = [np.array(f.by_col[var]) for var in varnames] + create a contiguity matrix from an external gal file + >>> w = libpysal.io.open(libpysal.examples.get_path("sids2.gal")).read() + create an instance of Moran_BV_matrix + >>> from esda.moran import Moran_BV_matrix >>> res = Moran_BV_matrix(vars, w, varnames = varnames) + check values + >>> round(res[(0, 1)].I,7) 0.1936261 >>> round(res[(3, 0)].I,7) 0.3770138 + """ try: # check if pandas is installed @@ -875,9 +889,10 @@ class Moran_Local(object): value to use as a weight for the "fake" neighbor for every island. If numpy.nan, will propagate to the final local statistic depending on the `stat_func`. If 0, then the lag is always zero for islands. - alternative: string - possible values -> "one-tailed"/ "two-tailed" - default value ->"two-tailed" + alternative: {"two-tailed", "one-tailed"} + default="two-tailed") + "two-tailed" test for the possibility of the relationship in both directions + "one-tailed" to detect an effect in one direction by not testing the effect in the other direction Attributes ---------- From f3c9a1c24a6b6cb4e84d9bbd203408868edd9d68 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Tue, 12 Apr 2022 16:27:46 +0530 Subject: [PATCH 7/9] added test coverage for local_moran --- esda/tests/test_moran.py | 54 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/esda/tests/test_moran.py b/esda/tests/test_moran.py index 65319885..f8bbd470 100644 --- a/esda/tests/test_moran.py +++ b/esda/tests/test_moran.py @@ -118,6 +118,19 @@ def test_Moran_Local(self): self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + def test_Moran_Local_two_tailed(self): + lm = moran.Moran_Local( + self.y, + self.w, + transformation="r", + permutations=99, + keep_simulations=True, + seed=SEED, + alternative="two-tailed" + ) + self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) + self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + def test_Moran_Local_parallel(self): lm = moran.Moran_Local( self.y, @@ -131,6 +144,47 @@ def test_Moran_Local_parallel(self): self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + def test_Moran_Local_two_tailed_parallel(self): + lm = moran.Moran_Local( + self.y, + self.w, + transformation="r", + n_jobs=-1, + permutations=99, + keep_simulations=True, + seed=SEED, + alternative="two-tailed" + ) + self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) + self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + + def test_Moran_Local_one_tailed(self): + lm = moran.Moran_Local( + self.y, + self.w, + transformation="r", + permutations=99, + keep_simulations=True, + seed=SEED, + alternative="one-tailed", + ) + self.assertAlmostEqual(lm.z_sim[0], -1.398058232171028) + self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + + def test_Moran_Local_one_tailed_parallel(self): + lm = moran.Moran_Local( + self.y, + self.w, + transformation="r", + n_jobs=-1, + permutations=99, + keep_simulations=True, + seed=SEED, + alternative="one-tailed" + ) + self.assertAlmostEqual(lm.z_sim[0], -1.398058232171028) + #self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + @unittest.skip("This function is being deprecated in the next release.") def test_by_col(self): From 1ae6263dc94013c471e4c5b416ccbe962397332c Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Tue, 12 Apr 2022 16:36:31 +0530 Subject: [PATCH 8/9] changed test --- esda/tests/test_moran.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esda/tests/test_moran.py b/esda/tests/test_moran.py index f8bbd470..bb5e8715 100644 --- a/esda/tests/test_moran.py +++ b/esda/tests/test_moran.py @@ -168,7 +168,7 @@ def test_Moran_Local_one_tailed(self): seed=SEED, alternative="one-tailed", ) - self.assertAlmostEqual(lm.z_sim[0], -1.398058232171028) + self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) def test_Moran_Local_one_tailed_parallel(self): @@ -182,8 +182,8 @@ def test_Moran_Local_one_tailed_parallel(self): seed=SEED, alternative="one-tailed" ) - self.assertAlmostEqual(lm.z_sim[0], -1.398058232171028) - #self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) + self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) @unittest.skip("This function is being deprecated in the next release.") From 68595a3875e9545bf8e1007a231461aa4a89b334 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Tue, 12 Apr 2022 17:25:49 +0530 Subject: [PATCH 9/9] changed test for single-tailed --- esda/tests/test_moran.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esda/tests/test_moran.py b/esda/tests/test_moran.py index bb5e8715..125877a1 100644 --- a/esda/tests/test_moran.py +++ b/esda/tests/test_moran.py @@ -169,7 +169,7 @@ def test_Moran_Local_one_tailed(self): alternative="one-tailed", ) self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) - self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + self.assertAlmostEqual(lm.p_z_sim[0], 0.4845338350758279) def test_Moran_Local_one_tailed_parallel(self): lm = moran.Moran_Local( @@ -183,7 +183,7 @@ def test_Moran_Local_one_tailed_parallel(self): alternative="one-tailed" ) self.assertAlmostEqual(lm.z_sim[0], -0.6990291160835514) - self.assertAlmostEqual(lm.p_z_sim[0], 0.24226691753791396) + self.assertAlmostEqual(lm.p_z_sim[0], 0.4845338350758279) @unittest.skip("This function is being deprecated in the next release.")