From fd69cb8d47dab90ea87d8fe31480c5c9ff96082d Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Fri, 6 Dec 2019 15:06:50 -0500 Subject: [PATCH 01/43] add beta iv estimation --- Projects/Project_1_HendricksLeukhina.ipynb | 85 ++++++++++++++++++++-- Projects/project_2_packages/iv_init.py | 44 +++++++++++ 2 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 Projects/project_2_packages/iv_init.py diff --git a/Projects/Project_1_HendricksLeukhina.ipynb b/Projects/Project_1_HendricksLeukhina.ipynb index 93013a4..75a665b 100644 --- a/Projects/Project_1_HendricksLeukhina.ipynb +++ b/Projects/Project_1_HendricksLeukhina.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -99,9 +99,34 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 80, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOkUlEQVR4nO3dXYxdV3nG8f/TmAgILYmT6ciNScdVrERRpSR0lCYCoTYmVSgR8UWUBlE0Qq58Q9vQUlHDHVIvEqni46JCsgh0LighNUG2QIJaJqitVKWMSVqSGOSQxmDLHwPE5aMSwfD24mzjyXg853jmnJlZnv9PGp291t7H59X2nsfL6+yPVBWSpPb82moXIElaGgNckhplgEtSowxwSWqUAS5Jjdqwkh92zTXX1MTExEp+pCQ17+DBg9+vqrH5/Ssa4BMTE8zMzKzkR0pS85IcWajfKRRJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWrUil6JKUmXgoldX7rguhcfevuK1eEIXJIa1TfAk9yQ5Ok5Pz9K8r4kG5PsT3K4e71qJQqWJPX0DfCq+nZV3VJVtwC/B/wf8AVgF3CgqrYCB7q2JGmFXOwUyjbgO1V1BLgXmO76p4HtwyxMkrS4iw3wB4DPdsvjVXW8Wz4BjC/0hiQ7k8wkmZmdnV1imZKk+QYO8CSXA+8A/nn+uqoqoBZ6X1XtrqrJqpocGzvvfuSSpCW6mBH424BvVNXJrn0yySaA7vXUsIuTJF3YxQT4Ozk3fQKwD5jqlqeAvcMqSpLU30ABnuQK4C7g8TndDwF3JTkMvLVrS5JWyEBXYlbVT4Gr5/X9gN5ZKZKkVeCVmJLUKANckhplgEtSowxwSWqUAS5JjTLAJalRPtBB0pq2Vh6e0M+Rh+8BIA+fv653t5HhcwQuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQX8kh6hZYunFnoohkY3YUza40jcElqlAEuSY0ywCWpUQa4JDVq0KfSX5lkT5JvJTmU5I4kG5PsT3K4e71q1MVKks4ZdAT+ceDLVXUjcDNwCNgFHKiqrcCBri1JWiF9AzzJ64G3AI8AVNXLVXUauBeY7jabBraPqkhJ0vkGOQ98CzALfDrJzcBB4EFgvKqOd9ucAMYXenOSncBOgOuuu27ZBUuXksXOuYa1c971Yg8rgPVz3vVaM8gUygbgjcAnqupW4KfMmy6p3t/egn+DVbW7qiaranJsbGy59UqSOoME+FHgaFU92bX30Av0k0k2AXSvp0ZToiRpIX0DvKpOAN9LckPXtQ14DtgHTHV9U8DekVQoSVrQoPdC+QvgM0kuB14A3kMv/B9LsgM4Atw/mhIlSQsZKMCr6mlgcoFV24ZbjiRpUF6JKUmNMsAlqVEGuCQ1ygc6aF1p6WEFsPCFM140o7McgUtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CjPA9e658MK1CpH4JLUKEfgGplWHhcmtcoRuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWrUQKcRJnkR+DHwC+BMVU0m2Qh8DpgAXgTur6qXRlOmLiU+rEAajosZgf9hVd1SVWefTr8LOFBVW4EDXVuStEKWM4VyLzDdLU8D25dfjiRpUIMGeAH/kuRgkp1d33hVHe+WTwDjC70xyc4kM0lmZmdnl1muJOmsQS+lf3NVHUvym8D+JN+au7KqKsmCk5dVtRvYDTA5OekEpyQNyUAj8Ko61r2eAr4A3AacTLIJoHs9NaoiJUnn6xvgSa5I8utnl4E/Ap4B9gFT3WZTwN5RFSlJOt8gUyjjwBeSnN3+n6rqy0m+DjyWZAdwBLh/dGVKkubrG+BV9QJw8wL9PwC2jaIoSVJ/3g/8ErLY/be997Z06fFSeklqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGuV54Je4xR6eAD5AQWqZI3BJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSo7yQZ8i6R88tyItmJA2TI3BJapQBLkmNGngKJcllwAxwrKruSbIFeBS4GjgIvLuqXh5NmWvPYs+flKSVcDEj8AeBQ3PaDwMfrarrgZeAHcMsTJK0uIECPMlm4O3AJ7t2gDuBPd0m08D2URQoSVrYoCPwjwEfAH7Zta8GTlfVma59FLh2oTcm2ZlkJsnM7OzssoqVJJ3TN8CT3AOcqqqDS/mAqtpdVZNVNTk2NraUP0KStIBBvsR8E/COJH8MvBr4DeDjwJVJNnSj8M3AsdGVKUmar+8IvKo+WFWbq2oCeAD4alW9C3gCuK/bbArYO7IqJUnnWc554H8L/HWS5+nNiT8ynJIkSYO4qEvpq+prwNe65ReA24ZfkiRpEF6JKUmNMsAlqVEGuCQ1ygCXpEYZ4JLUqOYf6OADFCStV47AJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGtU3wJO8Osl/JvmvJM8m+XDXvyXJk0meT/K5JJePvlxJ0lmDjMB/BtxZVTcDtwB3J7kdeBj4aFVdD7wE7BhdmZKk+foGePX8pGu+qvsp4E5gT9c/DWwfSYWSpAUNNAee5LIkTwOngP3Ad4DTVXWm2+QocO1oSpQkLWSgAK+qX1TVLcBm4DbgxkE/IMnOJDNJZmZnZ5dYpiRpvos6C6WqTgNPAHcAVyY5+0i2zcCxC7xnd1VNVtXk2NjYsoqVJJ3T95mYScaAn1fV6SSvAe6i9wXmE8B9wKPAFLB3lIVO7PrSKP94SWrOIA813gRMJ7mM3oj9sar6YpLngEeT/B3wFPDICOuUJM3TN8Cr6r+BWxfof4HefLgkaRV4JaYkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhrVN8CTvCHJE0meS/Jskge7/o1J9ic53L1eNfpyJUlnDTICPwO8v6puAm4H3pvkJmAXcKCqtgIHurYkaYX0DfCqOl5V3+iWfwwcAq4F7gWmu82mge2jKlKSdL6LmgNPMgHcCjwJjFfV8W7VCWD8Au/ZmWQmyczs7OwySpUkzTVwgCd5HfB54H1V9aO566qqgFrofVW1u6omq2pybGxsWcVKks4ZKMCTvIpeeH+mqh7vuk8m2dSt3wScGk2JkqSFDHIWSoBHgENV9ZE5q/YBU93yFLB3+OVJki5kwwDbvAl4N/DNJE93fR8CHgIeS7IDOALcP5oSJUkL6RvgVfXvQC6wettwy5EkDcorMSWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1Ki+AZ7kU0lOJXlmTt/GJPuTHO5erxptmZKk+QYZgf8jcPe8vl3AgaraChzo2pKkFdQ3wKvqX4Efzuu+F5julqeB7UOuS5LUx1LnwMer6ni3fAIYv9CGSXYmmUkyMzs7u8SPkyTNt+wvMauqgFpk/e6qmqyqybGxseV+nCSps9QAP5lkE0D3emp4JUmSBrHUAN8HTHXLU8De4ZQjSRrUIKcRfhb4D+CGJEeT7AAeAu5Kchh4a9eWJK2gDf02qKp3XmDVtiHXIkm6CF6JKUmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRi0rwJPcneTbSZ5PsmtYRUmS+ltygCe5DPgH4G3ATcA7k9w0rMIkSYtbzgj8NuD5qnqhql4GHgXuHU5ZkqR+NizjvdcC35vTPgr8/vyNkuwEdnbNnyT59jI+86IkWamPWsw1wPdhzdTzCmukJvfR4n61f2BN1HOeNVDTmt5HQ6jntxfqXE6AD6SqdgO7R/05a1WSmaqaXO061jL30eLcP/2t1320nCmUY8Ab5rQ3d32SpBWwnAD/OrA1yZYklwMPAPuGU5YkqZ8lT6FU1Zkkfw58BbgM+FRVPTu0yi4d63b66CK4jxbn/ulvXe6jVNVq1yBJWgKvxJSkRhngktQoA3xIkrwhyRNJnkvybJIHu/6NSfYnOdy9XrXata62JJcleSrJF7v2liRPdrdk+Fz3pfi6leTKJHuSfCvJoSR3eBy9UpK/6n7Pnkny2SSvXo/HkQE+PGeA91fVTcDtwHu7WwvsAg5U1VbgQNde7x4EDs1pPwx8tKquB14CdqxKVWvHx4EvV9WNwM309pXHUSfJtcBfApNV9bv0TqJ4gHV4HBngQ1JVx6vqG93yj+n90l1L7/YC091m08D21alwbUiyGXg78MmuHeBOYE+3ybreR0leD7wFeASgql6uqtN4HM23AXhNkg3Aa4HjrMPjyAAfgSQTwK3Ak8B4VR3vVp0AxleprLXiY8AHgF927auB01V1pmsfpfcP33q1BZgFPt1NM30yyRV4HP1KVR0D/h74Lr3g/l/gIOvwODLAhyzJ64DPA++rqh/NXVe9czbX7XmbSe4BTlXVwdWuZQ3bALwR+ERV3Qr8lHnTJR5HuYre/0i2AL8FXAHcvapFrRIDfIiSvIpeeH+mqh7vuk8m2dSt3wScWq361oA3Ae9I8iK9u1feSW++98ruv8LgLRmOAker6smuvYdeoHscnfNW4H+qaraqfg48Tu/YWnfHkQE+JN1c7iPAoar6yJxV+4CpbnkK2LvSta0VVfXBqtpcVRP0vnT6alW9C3gCuK/bbL3voxPA95Lc0HVtA57D42iu7wK3J3lt93t3dh+tu+PIKzGHJMmbgX8Dvsm5+d0P0ZsHfwy4DjgC3F9VP1yVIteQJH8A/E1V3ZPkd+iNyDcCTwF/WlU/W836VlOSW+h9yXs58ALwHnqDLY+jTpIPA39C7+yvp4A/ozfnva6OIwNckhrlFIokNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY36f0VxyriClAO2AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "# Parameters to use in simulation\n", "gamma_min = 0.35\n", @@ -120,7 +145,28 @@ "x = np.linspace(10, 90, 9)\n", "\n", "ax.bar(x-1.5, y2_totalcredit_deciles, width=2.)\n", - "ax.bar(x+1.50, y2_totalcredit_deciles, width=2.)" + "#ax.bar(x+1.50, y2_totalcredit_deciles, width=2., color = \"red\")\n", + "\n", + "# Simulation\n", + "tc = 6 \n", + "ncoursesattempted = 12\n", + "ncreditspercourse = 3\n", + "ncreditsgrad = 125\n", + "count = 25000\n", + "modelprob = tpy.HeterogeneousProbabilityModel(gamma_min, gamma_1, gamma_2, sigma, tc,\n", + " ncoursesattempted, ncreditspercourse, ncreditsgrad)\n", + "ability, gpa, credits = modelprob.simulate(count)\n", + "twoyears = []\n", + "for array in credits:\n", + " twoyears.append(array[0] + array[1])\n", + "assert len(twoyears) == count\n", + "deciles = []\n", + "xax = []\n", + "for i in range(10,100,10):\n", + " xax.append(i)\n", + " deciles.append(np.percentile(twoyears, i))\n", + "ax.bar(xax, deciles, width = 2, color = \"black\")\n", + "#plt.scatter(deciles, deciles)" ] }, { @@ -136,9 +182,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOcUlEQVR4nO3dX4wd5X3G8e9THJSEtIBhu3Jx6LqKBUKVgHRFQYmiFoeKFBR8gShRilaRI98kLbSpUidXROoFkar8uagiWSHpXqQB6oBsJRKt5RC1lSqadaANYJAJhcSW/2wSHCiVQpz8enHG8nZZ7znePWfXL/v9SNaZ952ZPT+NZh/PvmfeOakqJEnt+bXVLkCStDQGuCQ1ygCXpEYZ4JLUKANckhq1biXf7NJLL62JiYmVfEtJat7+/ft/XFVj8/tXNMAnJiaYmZlZybeUpOYleWmhfodQJKlRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUSs6E1OSWjOx41tnXPfifbecbtx74Zl/yL0/G2JFp3kFLkmN6hvgSa5I8uScf68kuSfJ+iR7kxzsXi9eiYIlST19A7yqnquqa6rqGuD3gP8FHgF2APuqajOwr2tLklbI2Q6hbAF+UFUvAbcB013/NLB1mIVJkhZ3tgF+J/D1bnm8qo50y0eB8YV2SLI9yUySmdnZ2SWWKUmab+AAT3I+8EHgH+evq6oCaqH9qmpnVU1W1eTY2BueRy5JWqKzuQL/APC9qjrWtY8l2QDQvR4fdnGSpDM7mwD/EKeHTwD2AFPd8hSwe1hFSZL6GyjAk1wA3AQ8PKf7PuCmJAeB93dtSdIKGWgmZlW9Blwyr+8n9O5KkSStAmdiSlKjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEb5hQ6SzilD+QIFGNmXKJxLvAKXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcqJPNIaN/DEGVh88swamDhzrvEKXJIaZYBLUqMMcElqlAEuSY0a9FvpL0qyK8mzSQ4kuSHJ+iR7kxzsXi8edbGSpNMGvQL/IvBoVV0JXA0cAHYA+6pqM7Cva0uSVkjfAE9yIfA+4H6Aqnq9qk4AtwHT3WbTwNZRFSlJeqNB7gPfBMwCX01yNbAfuBsYr6oj3TZHgfGFdk6yHdgOcPnlly+7YKl13netYRlkCGUd8G7gS1V1LfAa84ZLqqqAWmjnqtpZVZNVNTk2NrbceiVJnUEC/BBwqKoe79q76AX6sSQbALrX46MpUZK0kL4BXlVHgR8luaLr2gI8A+wBprq+KWD3SCqUJC1o0Geh/BnwtSTnAy8AH6EX/g8l2Qa8BNwxmhIlSQsZKMCr6klgcoFVW4ZbjiRpUM7ElKRGGeCS1CgDXJIa5Rc66E3PiTN6s/IKXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGuVMTA3dYjMfYd7sR2c+SkvmFbgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElq1EC3ESZ5EXgV+CVwsqomk6wHHgQmgBeBO6rq5dGUKUma72yuwP+wqq6pqlPfTr8D2FdVm4F9XVuStEKWM4RyGzDdLU8DW5dfjiRpUIMGeAH/nGR/ku1d33hVHemWjwLjC+2YZHuSmSQzs7OzyyxXknTKoFPp31tVh5P8JrA3ybNzV1ZVJamFdqyqncBOgMnJyQW3kSSdvYGuwKvqcPd6HHgEuA44lmQDQPd6fFRFSpLeqG+AJ7kgya+fWgb+CHgK2ANMdZtNAbtHVaQk6Y0GGUIZBx5Jcmr7f6iqR5N8F3goyTbgJeCO0ZUpSZqvb4BX1QvA1Qv0/wTYMoqiJEn9+TzwN4nFnsE98PO3wWdwSw1xKr0kNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUqIG/EzPJecAMcLiqbk2yCXgAuATYD9xVVa+Ppsxzz8DfQQmLfw+l30EpaYnO5gr8buDAnPZngc9X1buAl4FtwyxMkrS4gQI8yUbgFuDLXTvAjcCubpNpYOsoCpQkLWzQK/AvAJ8EftW1LwFOVNXJrn0IuGyhHZNsTzKTZGZ2dnZZxUqSTusb4EluBY5X1f6lvEFV7ayqyaqaHBsbW8qPkCQtYJAPMd8DfDDJHwNvBX4D+CJwUZJ13VX4RuDw6MqUJM3X9wq8qj5VVRuragK4E/h2VX0YeAy4vdtsCtg9siolSW+wnPvA/xr4yyTP0xsTv384JUmSBjHwfeAAVfUd4Dvd8gvAdcMvSZI0CGdiSlKjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY3qG+BJ3prkP5L8Z5Knk3ym69+U5PEkzyd5MMn5oy9XknTKIFfgPwdurKqrgWuAm5NcD3wW+HxVvQt4Gdg2ujIlSfP1DfDq+Z+u+ZbuXwE3Aru6/mlg60gqlCQtaKAx8CTnJXkSOA7sBX4AnKiqk90mh4DLRlOiJGkhAwV4Vf2yqq4BNgLXAVcO+gZJtieZSTIzOzu7xDIlSfOd1V0oVXUCeAy4Abgoybpu1Ubg8Bn22VlVk1U1OTY2tqxiJUmnreu3QZIx4BdVdSLJ24Cb6H2A+RhwO/AAMAXsHmWhEzu+dcZ1L953y//vuPfCM/+ge382pIokaXX1DXBgAzCd5Dx6V+wPVdU3kzwDPJDkb4AngPtHWKckaZ6+AV5V/wVcu0D/C/TGwyVJq8CZmJLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJalTfAE/yziSPJXkmydNJ7u761yfZm+Rg93rx6MuVJJ0yyBX4SeATVXUVcD3wsSRXATuAfVW1GdjXtSVJK6RvgFfVkar6Xrf8KnAAuAy4DZjuNpsGto6qSEnSG53VGHiSCeBa4HFgvKqOdKuOAuNn2Gd7kpkkM7Ozs8soVZI018ABnuQdwDeAe6rqlbnrqqqAWmi/qtpZVZNVNTk2NrasYiVJpw0U4EneQi+8v1ZVD3fdx5Js6NZvAI6PpkRJ0kIGuQslwP3Agar63JxVe4CpbnkK2D388iRJZ7JugG3eA9wFfD/Jk13fp4H7gIeSbANeAu4YTYmSpIX0DfCq+jcgZ1i9ZbjlSJIG5UxMSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1qm+AJ/lKkuNJnprTtz7J3iQHu9eLR1umJGm+Qa7A/x64eV7fDmBfVW0G9nVtSdIK6hvgVfUvwE/ndd8GTHfL08DWIdclSepjqWPg41V1pFs+CoyfacMk25PMJJmZnZ1d4ttJkuZb9oeYVVVALbJ+Z1VNVtXk2NjYct9OktRZaoAfS7IBoHs9PrySJEmDWGqA7wGmuuUpYPdwypEkDWqQ2wi/Dvw7cEWSQ0m2AfcBNyU5CLy/a0uSVtC6fhtU1YfOsGrLkGuRJJ0FZ2JKUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRywrwJDcneS7J80l2DKsoSVJ/Sw7wJOcBfwd8ALgK+FCSq4ZVmCRpccu5Ar8OeL6qXqiq14EHgNuGU5YkqZ9U1dJ2TG4Hbq6qj3btu4Dfr6qPz9tuO7C9a14BPLf0cptzKfDj1S7iHOWxWZzHZ3Fr7fj8dlWNze9cN+p3raqdwM5Rv8+5KMlMVU2udh3nIo/N4jw+i/P49CxnCOUw8M457Y1dnyRpBSwnwL8LbE6yKcn5wJ3AnuGUJUnqZ8lDKFV1MsnHgX8CzgO+UlVPD62yN4c1OXQ0II/N4jw+i/P4sIwPMSVJq8uZmJLUKANckhplgA9BkncmeSzJM0meTnJ3178+yd4kB7vXi1e71tWU5LwkTyT5ZtfelOTx7lEMD3Yfhq85SS5KsivJs0kOJLnBc+e0JH/R/V49leTrSd7qudNjgA/HSeATVXUVcD3wse6xAjuAfVW1GdjXtdeyu4EDc9qfBT5fVe8CXga2rUpVq++LwKNVdSVwNb1j5LkDJLkM+HNgsqp+l94NE3fiuQMY4ENRVUeq6nvd8qv0fgEvo/dogelus2lg6+pUuPqSbARuAb7ctQPcCOzqNlmTxyfJhcD7gPsBqur1qjqB585c64C3JVkHvB04gucOYIAPXZIJ4FrgcWC8qo50q44C46tU1rngC8AngV917UuAE1V1smsfovef3lqzCZgFvtoNL305yQV47gBQVYeBvwV+SC+4fwbsx3MHMMCHKsk7gG8A91TVK3PXVe9+zTV5z2aSW4HjVbV/tWs5B60D3g18qaquBV5j3nDJGj93Lqb318gm4LeAC4CbV7Woc4gBPiRJ3kIvvL9WVQ933ceSbOjWbwCOr1Z9q+w9wAeTvEjvqZU30hv3vaj7sxjW7qMYDgGHqurxrr2LXqB77vS8H/jvqpqtql8AD9M7nzx3MMCHohvPvR84UFWfm7NqDzDVLU8Bu1e6tnNBVX2qqjZW1QS9D6C+XVUfBh4Dbu82W5PHp6qOAj9KckXXtQV4Bs+dU34IXJ/k7d3v2anjs+bPHXAm5lAkeS/wr8D3OT3G+2l64+APAZcDLwF3VNVPV6XIc0SSPwD+qqpuTfI79K7I1wNPAH9aVT9fzfpWQ5Jr6H24ez7wAvARehdXnjtAks8Af0Lvbq8ngI/SG/P23DHAJalNDqFIUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktSo/wOs0cBzEAeD1wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "# Deciles from the data\n", "y2_totalcredit_deciles = np.array([\n", @@ -193,7 +262,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/Projects/project_2_packages/iv_init.py b/Projects/project_2_packages/iv_init.py new file mode 100644 index 0000000..271a35e --- /dev/null +++ b/Projects/project_2_packages/iv_init.py @@ -0,0 +1,44 @@ +''' + +Python Package to implement instrumental variables +Author: Group 2 NYU Pre docs +Date Created: December 6, 2019 + +''' + +import numpy as np + +mat_x = np.transpose(np.matrix([ [4, .2, 1, 5], + [.4, 2, 2, 2], + [0, 0, .3, 1]])) + +mat_z = np.transpose(np.matrix([[1, .5, 2, 3], + [3, 0, .2, 3], + [1, 0, .4, 3]])) + +mat_y = np.transpose(np.array([[3, 5, 2, 1]])) + +def projection_matrix(b): + ''' + Inputs: + a (matrix) -- x + b (matrix) -- z + ''' + P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) + return P_b + +def estimate_beta_iv(a, b, c): + ''' + + ''' + + proj = projection_matrix(b) + b_1 = np.transpose(a) @ proj @ a + b_2 = np.linalg.inv(b_1) + b_3 = np.transpose(a) @ proj @ c + return b_2 @ b_3 + + #b_iv = np.linalg.inv(np.transpose(a) @ projection_matrix(b) @ a) \ + # @ np.transpose(a) @ projection_matrix(b) @ c + +print(estimate_beta_iv(mat_x, mat_z, mat_y)) From ebb04a8f0261bcdbe9fcb38062d68c5e08ae324e Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 15:08:41 -0500 Subject: [PATCH 02/43] added SEs --- .../project_2_packages/iv_standard_error.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 Projects/project_2_packages/iv_standard_error.py diff --git a/Projects/project_2_packages/iv_standard_error.py b/Projects/project_2_packages/iv_standard_error.py new file mode 100644 index 0000000..a241b0f --- /dev/null +++ b/Projects/project_2_packages/iv_standard_error.py @@ -0,0 +1,24 @@ + + +import numpy as np + +Z = np.matrix([[1, 2, 3], + [4, 5, 6], + [7, 8, 9], + [10, 11, 12]]) + + + +def calculate_sigma(Z, X, Y, beta_iv): + N = Z.shape[0] + resid = Y - X @ beta_iv + + sigma_iv = (1/N) * np.transpose(resid) @ resid + + return sigma_iv + + + +def calculate_var_beta(sigma, X, Z): + var_beta = sigma^2 * np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) + return var_beta \ No newline at end of file From 4b27b55bfcb8f611344eafb62fc08c81e1915e03 Mon Sep 17 00:00:00 2001 From: Nicolaj Thor Date: Fri, 6 Dec 2019 15:21:31 -0500 Subject: [PATCH 03/43] Add setup files' --- Projects/project_2_packages/LICENSE.MD | 21 +++++++++++++++++++++ Projects/project_2_packages/README.MD | 3 +++ Projects/project_2_packages/setup.py | 16 ++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 Projects/project_2_packages/LICENSE.MD create mode 100644 Projects/project_2_packages/README.MD create mode 100644 Projects/project_2_packages/setup.py diff --git a/Projects/project_2_packages/LICENSE.MD b/Projects/project_2_packages/LICENSE.MD new file mode 100644 index 0000000..f131108 --- /dev/null +++ b/Projects/project_2_packages/LICENSE.MD @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) [2019] [Best Group 2] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Projects/project_2_packages/README.MD b/Projects/project_2_packages/README.MD new file mode 100644 index 0000000..bbc2d26 --- /dev/null +++ b/Projects/project_2_packages/README.MD @@ -0,0 +1,3 @@ +# iv-reg package + +This is a simple package to perform instrumental variable estimation. diff --git a/Projects/project_2_packages/setup.py b/Projects/project_2_packages/setup.py new file mode 100644 index 0000000..2703e37 --- /dev/null +++ b/Projects/project_2_packages/setup.py @@ -0,0 +1,16 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="iv-reg", # Replace with your own username + version="0.0.1", + author="Best Group 2", + description="Package for Instrumental Variable Estimation", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/jettpettus/modularizationandtesting", + packages=setuptools.find_packages(), + python_requires='>=3.6', +) From 6f17916df1acc8f940e98d82608d2cc31fcdaec8 Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Fri, 6 Dec 2019 15:22:00 -0500 Subject: [PATCH 04/43] Added doc strings and checks --- Projects/project_2_packages/iv_init.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Projects/project_2_packages/iv_init.py b/Projects/project_2_packages/iv_init.py index 271a35e..4e66c26 100644 --- a/Projects/project_2_packages/iv_init.py +++ b/Projects/project_2_packages/iv_init.py @@ -18,18 +18,34 @@ mat_y = np.transpose(np.array([[3, 5, 2, 1]])) +def check_dim(a, b, c): + try: + assert a.shape[1] == b.shape[1] == c.shape[1] + except: + raise Exception('Matrices are the wrong shape') + + try: + assert a.shape[1] <= b.shape[1] + except: + raise Exception('Model underidentified') + + def projection_matrix(b): ''' Inputs: a (matrix) -- x b (matrix) -- z ''' + P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) return P_b def estimate_beta_iv(a, b, c): ''' - + Inputs: + a (matrix) -- x + b (matrix) -- z + c (matrix) -- y ''' proj = projection_matrix(b) @@ -38,7 +54,4 @@ def estimate_beta_iv(a, b, c): b_3 = np.transpose(a) @ proj @ c return b_2 @ b_3 - #b_iv = np.linalg.inv(np.transpose(a) @ projection_matrix(b) @ a) \ - # @ np.transpose(a) @ projection_matrix(b) @ c - print(estimate_beta_iv(mat_x, mat_z, mat_y)) From c1cdf1b5248321a2f5b20dbfe3cf59517786119e Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 15:31:35 -0500 Subject: [PATCH 05/43] moved directory --- .../project_2_packages/iv_jett/__init__.py | 0 .../project_2_packages/iv_jett/iv_init.py | 32 +++++++++++++++++++ .../iv_jett/iv_standard_error.py | 17 ++++++++++ 3 files changed, 49 insertions(+) create mode 100644 Projects/project_2_packages/iv_jett/__init__.py create mode 100644 Projects/project_2_packages/iv_jett/iv_init.py create mode 100644 Projects/project_2_packages/iv_jett/iv_standard_error.py diff --git a/Projects/project_2_packages/iv_jett/__init__.py b/Projects/project_2_packages/iv_jett/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Projects/project_2_packages/iv_jett/iv_init.py b/Projects/project_2_packages/iv_jett/iv_init.py new file mode 100644 index 0000000..fb068e2 --- /dev/null +++ b/Projects/project_2_packages/iv_jett/iv_init.py @@ -0,0 +1,32 @@ +''' + +Python Package to implement instrumental variables +Author: Group 2 NYU Pre docs +Date Created: December 6, 2019 + +''' +import numpy as np + + +def projection_matrix(b): + ''' + Inputs: + a (matrix) -- x + b (matrix) -- z + ''' + P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) + return P_b + +def estimate_beta_iv(a, b, c): + ''' + + ''' + + proj = projection_matrix(b) + b_1 = np.transpose(a) @ proj @ a + b_2 = np.linalg.inv(b_1) + b_3 = np.transpose(a) @ proj @ c + return b_2 @ b_3 + + #b_iv = np.linalg.inv(np.transpose(a) @ projection_matrix(b) @ a) \ + # @ np.transpose(a) @ projection_matrix(b) @ c diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py new file mode 100644 index 0000000..c397488 --- /dev/null +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -0,0 +1,17 @@ +import numpy as np + + + +def calculate_sigma(Z, X, Y, beta_iv): + N = Z.shape[0] + resid = Y - X @ beta_iv + + sigma_iv = (1/N) * np.transpose(resid) @ resid + + return sigma_iv + + + +def calculate_var_beta(sigma, X, Z): + var_beta = sigma^2 * np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) + return var_beta \ No newline at end of file From 26075e3cde45942759f5bfa969a0286e45d3d597 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 15:32:00 -0500 Subject: [PATCH 06/43] deleted as moved into new folder --- Projects/project_2_packages/iv_init.py | 44 ------------------- .../project_2_packages/iv_standard_error.py | 7 --- 2 files changed, 51 deletions(-) delete mode 100644 Projects/project_2_packages/iv_init.py diff --git a/Projects/project_2_packages/iv_init.py b/Projects/project_2_packages/iv_init.py deleted file mode 100644 index 271a35e..0000000 --- a/Projects/project_2_packages/iv_init.py +++ /dev/null @@ -1,44 +0,0 @@ -''' - -Python Package to implement instrumental variables -Author: Group 2 NYU Pre docs -Date Created: December 6, 2019 - -''' - -import numpy as np - -mat_x = np.transpose(np.matrix([ [4, .2, 1, 5], - [.4, 2, 2, 2], - [0, 0, .3, 1]])) - -mat_z = np.transpose(np.matrix([[1, .5, 2, 3], - [3, 0, .2, 3], - [1, 0, .4, 3]])) - -mat_y = np.transpose(np.array([[3, 5, 2, 1]])) - -def projection_matrix(b): - ''' - Inputs: - a (matrix) -- x - b (matrix) -- z - ''' - P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) - return P_b - -def estimate_beta_iv(a, b, c): - ''' - - ''' - - proj = projection_matrix(b) - b_1 = np.transpose(a) @ proj @ a - b_2 = np.linalg.inv(b_1) - b_3 = np.transpose(a) @ proj @ c - return b_2 @ b_3 - - #b_iv = np.linalg.inv(np.transpose(a) @ projection_matrix(b) @ a) \ - # @ np.transpose(a) @ projection_matrix(b) @ c - -print(estimate_beta_iv(mat_x, mat_z, mat_y)) diff --git a/Projects/project_2_packages/iv_standard_error.py b/Projects/project_2_packages/iv_standard_error.py index a241b0f..457c9c4 100644 --- a/Projects/project_2_packages/iv_standard_error.py +++ b/Projects/project_2_packages/iv_standard_error.py @@ -2,13 +2,6 @@ import numpy as np -Z = np.matrix([[1, 2, 3], - [4, 5, 6], - [7, 8, 9], - [10, 11, 12]]) - - - def calculate_sigma(Z, X, Y, beta_iv): N = Z.shape[0] resid = Y - X @ beta_iv From bafe70eabe097ba0793b17d1051eb742493f314f Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 15:36:17 -0500 Subject: [PATCH 07/43] added import statement --- Projects/project_2_packages/iv_jett/iv_standard_error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index c397488..8e8fc71 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -1,6 +1,6 @@ import numpy as np - +from iv_jett.iv_init import projection_matrix def calculate_sigma(Z, X, Y, beta_iv): N = Z.shape[0] From 3fa8df7a32c8336b25128caf53de3f5c455c7cfd Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Fri, 6 Dec 2019 15:43:49 -0500 Subject: [PATCH 08/43] More checks --- Projects/project_2_packages/iv_init.py | 28 +++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Projects/project_2_packages/iv_init.py b/Projects/project_2_packages/iv_init.py index 4e66c26..f7396a5 100644 --- a/Projects/project_2_packages/iv_init.py +++ b/Projects/project_2_packages/iv_init.py @@ -8,21 +8,29 @@ import numpy as np -mat_x = np.transpose(np.matrix([ [4, .2, 1, 5], - [.4, 2, 2, 2], - [0, 0, .3, 1]])) +mat_x = np.transpose(np.matrix([ [4, .2, 1, 5, 1, 10, 20], + [.4, 2, 2, 2, 2, 11, 25], + [0, 0, .3, 1, 3, 12, 40]])) -mat_z = np.transpose(np.matrix([[1, .5, 2, 3], - [3, 0, .2, 3], - [1, 0, .4, 3]])) +mat_z = np.transpose(np.matrix([[1, .5, 2, 3, 4, 13, 56], + [3, 0, .2, 3, 5, 14, 60], + [1, 0, .4, 3, 6, 15, 77]])) -mat_y = np.transpose(np.array([[3, 5, 2, 1]])) +mat_y = np.transpose(np.array([[3, 5, 2, 1, 7, 16, 80]])) def check_dim(a, b, c): + ''' + Check dimensions + ''' + try: + assert len(a.shape) == 2 & len(b.shape) == 2 & len(c.shape) == 2 + except: + raise Exception('Incorrect data dimension input') + try: - assert a.shape[1] == b.shape[1] == c.shape[1] + assert a.shape[0] == b.shape[0] == c.shape[0] except: - raise Exception('Matrices are the wrong shape') + raise Exception('Number of observations not equal') try: assert a.shape[1] <= b.shape[1] @@ -48,10 +56,12 @@ def estimate_beta_iv(a, b, c): c (matrix) -- y ''' + check_dim(a, b, c) proj = projection_matrix(b) b_1 = np.transpose(a) @ proj @ a b_2 = np.linalg.inv(b_1) b_3 = np.transpose(a) @ proj @ c return b_2 @ b_3 + print(estimate_beta_iv(mat_x, mat_z, mat_y)) From d6ab78a2785d09eb281cb30ae0c9d421622936e0 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 15:49:53 -0500 Subject: [PATCH 09/43] first working case of SE --- Projects/project_2_packages/iv_jett/iv_standard_error.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index 8e8fc71..c220726 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -13,5 +13,6 @@ def calculate_sigma(Z, X, Y, beta_iv): def calculate_var_beta(sigma, X, Z): - var_beta = sigma^2 * np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) - return var_beta \ No newline at end of file + var_beta = (np.asscalar(sigma**2) * np.identity(X.shape[1])) @ np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) + se_beta = np.sqrt(np.diag(var_beta)) + return se_beta \ No newline at end of file From 667851408360404e4f6ddf745adadac6e4d8a37a Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 15:50:15 -0500 Subject: [PATCH 10/43] added SEs to checks --- Projects/project_2_packages/data_test.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 Projects/project_2_packages/data_test.py diff --git a/Projects/project_2_packages/data_test.py b/Projects/project_2_packages/data_test.py new file mode 100644 index 0000000..213f990 --- /dev/null +++ b/Projects/project_2_packages/data_test.py @@ -0,0 +1,28 @@ +import numpy as np +from iv_jett.iv_standard_error import * +from iv_jett.iv_init import * + +mat_x = np.transpose(np.matrix([ [4, .2, 1, 5], + [.4, 2, 2, 2], + [0, 0, .3, 1]])) + +mat_z = np.transpose(np.matrix([[1, .5, 2, 3], + [3, 0, .2, 3], + [1, 0, .4, 3]])) + +mat_y = np.transpose(np.array([[3, 5, 2, 1]])) + + + +beta_iv_hat = estimate_beta_iv(mat_x, mat_z, mat_y) + +print(beta_iv_hat) + + +sigma = calculate_sigma(mat_z, mat_x, mat_y, beta_iv_hat) + +print(sigma) + + +var_beta = calculate_var_beta(sigma, mat_x, mat_z) +print(var_beta) \ No newline at end of file From 0806eca2f0b5882d329bef25a3426ff78f5cf8e8 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 15:55:33 -0500 Subject: [PATCH 11/43] added sqrt for sigma bug --- Projects/project_2_packages/data_test.py | 14 +++++++------- .../iv_jett/iv_standard_error.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Projects/project_2_packages/data_test.py b/Projects/project_2_packages/data_test.py index 213f990..317212b 100644 --- a/Projects/project_2_packages/data_test.py +++ b/Projects/project_2_packages/data_test.py @@ -2,15 +2,15 @@ from iv_jett.iv_standard_error import * from iv_jett.iv_init import * -mat_x = np.transpose(np.matrix([ [4, .2, 1, 5], - [.4, 2, 2, 2], - [0, 0, .3, 1]])) +mat_x = np.transpose(np.matrix([ [4, .2, 1, 5, 1, 10, 20], + [.4, 2, 2, 2, 2, 11, 25], + [0, 0, .3, 1, 3, 12, 40]])) -mat_z = np.transpose(np.matrix([[1, .5, 2, 3], - [3, 0, .2, 3], - [1, 0, .4, 3]])) +mat_z = np.transpose(np.matrix([[1, .5, 2, 3, 4, 13, 56], + [3, 0, .2, 3, 5, 14, 60], + [1, 0, .4, 3, 6, 15, 77]])) -mat_y = np.transpose(np.array([[3, 5, 2, 1]])) +mat_y = np.transpose(np.array([[3, 5, 2, 1, 7, 16, 80]])) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index c220726..cb1308b 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -8,7 +8,7 @@ def calculate_sigma(Z, X, Y, beta_iv): sigma_iv = (1/N) * np.transpose(resid) @ resid - return sigma_iv + return np.sqrt(sigma_iv) From fe9d8d5cd1696c6cb57c7b13225c55807b8847a6 Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Fri, 6 Dec 2019 16:07:49 -0500 Subject: [PATCH 12/43] Clean up Ed's code --- Projects/project_2_packages/iv_jett/iv_standard_error.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index cb1308b..9916fa8 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -1,5 +1,4 @@ import numpy as np - from iv_jett.iv_init import projection_matrix def calculate_sigma(Z, X, Y, beta_iv): @@ -10,9 +9,7 @@ def calculate_sigma(Z, X, Y, beta_iv): return np.sqrt(sigma_iv) - - def calculate_var_beta(sigma, X, Z): var_beta = (np.asscalar(sigma**2) * np.identity(X.shape[1])) @ np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) se_beta = np.sqrt(np.diag(var_beta)) - return se_beta \ No newline at end of file + return se_beta From bf0031d0ffd97c451c97e7e4b12a4047a2f2a437 Mon Sep 17 00:00:00 2001 From: Nicolas Jimenez Date: Fri, 6 Dec 2019 16:39:37 -0500 Subject: [PATCH 13/43] Updated constant option in iv_init --- Projects/project_2_packages/iv_init.py | 30 ++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/Projects/project_2_packages/iv_init.py b/Projects/project_2_packages/iv_init.py index f7396a5..03a83f3 100644 --- a/Projects/project_2_packages/iv_init.py +++ b/Projects/project_2_packages/iv_init.py @@ -37,7 +37,6 @@ def check_dim(a, b, c): except: raise Exception('Model underidentified') - def projection_matrix(b): ''' Inputs: @@ -48,20 +47,43 @@ def projection_matrix(b): P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) return P_b -def estimate_beta_iv(a, b, c): +def estimate_beta_iv(a, b, c, nocons=""): ''' Inputs: a (matrix) -- x b (matrix) -- z c (matrix) -- y ''' + if nocons != "" & nocons != "nocons": + raise Exception('nocons option misspecified') check_dim(a, b, c) + if nocons == "": + N = a.shape[0] + a_1 = np.ones((N,1)) + a = np.hstack((a_1,a)) + b_1 = np.ones((N,1)) + b = np.hstack((b_1, b)) + + proj = projection_matrix(b) b_1 = np.transpose(a) @ proj @ a b_2 = np.linalg.inv(b_1) b_3 = np.transpose(a) @ proj @ c - return b_2 @ b_3 + betas_all = b_2 @ b_3 + + if nocons == "": + cons = betas_all[0] + betas = betas_all[1:] + print("_cons: " + str(cons)) + for i in range(0, len(betas)): + print("beta_"+ str(i) + " :" + str(betas[i])) + else if nocons == "nocons": + betas = betas_all[0:] + for i in range(0, len(betas)): + print("beta_"+ str(i) + " :" + str(betas[i])) + return betas_all + +estimate_beta_iv(mat_x, mat_z, mat_y) -print(estimate_beta_iv(mat_x, mat_z, mat_y)) From c8a12c663642b20cc711409c5bb12d6a5e8f5e6d Mon Sep 17 00:00:00 2001 From: Nicolas Jimenez Date: Fri, 6 Dec 2019 16:47:02 -0500 Subject: [PATCH 14/43] test message ` --- Projects/project_2_packages/iv_jett/iv_init.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/iv_init.py b/Projects/project_2_packages/iv_jett/iv_init.py index 688afa5..b96fd8c 100644 --- a/Projects/project_2_packages/iv_jett/iv_init.py +++ b/Projects/project_2_packages/iv_jett/iv_init.py @@ -74,9 +74,3 @@ def estimate_beta_iv(a, b, c, nocons=""): print("beta_"+ str(i) + " :" + str(betas[i])) return betas_all -estimate_beta_iv(mat_x, mat_z, mat_y) - -<<<<<<< HEAD:Projects/project_2_packages/iv_init.py - -======= ->>>>>>> fe9d8d5cd1696c6cb57c7b13225c55807b8847a6:Projects/project_2_packages/iv_jett/iv_init.py From 85f5830b51eef6c80654682746c3ec12ffdc17da Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 16:52:14 -0500 Subject: [PATCH 15/43] removed else if statement --- Projects/project_2_packages/iv_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Projects/project_2_packages/iv_init.py b/Projects/project_2_packages/iv_init.py index 03a83f3..faac069 100644 --- a/Projects/project_2_packages/iv_init.py +++ b/Projects/project_2_packages/iv_init.py @@ -78,7 +78,7 @@ def estimate_beta_iv(a, b, c, nocons=""): print("_cons: " + str(cons)) for i in range(0, len(betas)): print("beta_"+ str(i) + " :" + str(betas[i])) - else if nocons == "nocons": + else nocons == "nocons": betas = betas_all[0:] for i in range(0, len(betas)): print("beta_"+ str(i) + " :" + str(betas[i])) From 4ae40503b1423db47ea607372b5523c2f272123c Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Fri, 6 Dec 2019 16:55:32 -0500 Subject: [PATCH 16/43] Ed has horrible documentation --- Projects/project_2_packages/iv_jett/iv_standard_error.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index 9916fa8..7e263b5 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -1,12 +1,14 @@ +''' +Calculate Covariance Matrix +''' + import numpy as np from iv_jett.iv_init import projection_matrix def calculate_sigma(Z, X, Y, beta_iv): N = Z.shape[0] resid = Y - X @ beta_iv - sigma_iv = (1/N) * np.transpose(resid) @ resid - return np.sqrt(sigma_iv) def calculate_var_beta(sigma, X, Z): From c2c39e0126e62174089b1839407ecb695e2bf49b Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Fri, 6 Dec 2019 16:56:46 -0500 Subject: [PATCH 17/43] incorrect else --- Projects/project_2_packages/iv_jett/iv_init.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/iv_init.py b/Projects/project_2_packages/iv_jett/iv_init.py index b96fd8c..891d122 100644 --- a/Projects/project_2_packages/iv_jett/iv_init.py +++ b/Projects/project_2_packages/iv_jett/iv_init.py @@ -49,7 +49,7 @@ def estimate_beta_iv(a, b, c, nocons=""): check_dim(a, b, c) if nocons == "": - N = a.shape[0] + N = a.shape[0] a_1 = np.ones((N,1)) a = np.hstack((a_1,a)) b_1 = np.ones((N,1)) @@ -68,9 +68,8 @@ def estimate_beta_iv(a, b, c, nocons=""): print("_cons: " + str(cons)) for i in range(0, len(betas)): print("beta_"+ str(i) + " :" + str(betas[i])) - else if nocons == "nocons": - betas = betas_all[0:] + elif nocons == "nocons": + betas = betas_all[0:] for i in range(0, len(betas)): print("beta_"+ str(i) + " :" + str(betas[i])) return betas_all - From 01719a75b801030fa125d6d22830bd59972519a6 Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Fri, 6 Dec 2019 16:57:57 -0500 Subject: [PATCH 18/43] dfdjkls --- Projects/project_2_packages/iv_jett/iv_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Projects/project_2_packages/iv_jett/iv_init.py b/Projects/project_2_packages/iv_jett/iv_init.py index 891d122..bdbb3ab 100644 --- a/Projects/project_2_packages/iv_jett/iv_init.py +++ b/Projects/project_2_packages/iv_jett/iv_init.py @@ -44,7 +44,7 @@ def estimate_beta_iv(a, b, c, nocons=""): b (matrix) -- z c (matrix) -- y ''' - if nocons != "" & nocons != "nocons": + if nocons != "" and nocons != "nocons": raise Exception('nocons option misspecified') check_dim(a, b, c) From ffd7855ba4701962d714cffee7b4ef486ee0b61b Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 23:02:49 -0500 Subject: [PATCH 19/43] these were moved into sub-directory but I think SOMEONE accidentally re-commited --- Projects/project_2_packages/iv_init.py | 89 ------------------- .../project_2_packages/iv_standard_error.py | 17 ---- 2 files changed, 106 deletions(-) delete mode 100644 Projects/project_2_packages/iv_init.py delete mode 100644 Projects/project_2_packages/iv_standard_error.py diff --git a/Projects/project_2_packages/iv_init.py b/Projects/project_2_packages/iv_init.py deleted file mode 100644 index faac069..0000000 --- a/Projects/project_2_packages/iv_init.py +++ /dev/null @@ -1,89 +0,0 @@ -''' - -Python Package to implement instrumental variables -Author: Group 2 NYU Pre docs -Date Created: December 6, 2019 - -''' - -import numpy as np - -mat_x = np.transpose(np.matrix([ [4, .2, 1, 5, 1, 10, 20], - [.4, 2, 2, 2, 2, 11, 25], - [0, 0, .3, 1, 3, 12, 40]])) - -mat_z = np.transpose(np.matrix([[1, .5, 2, 3, 4, 13, 56], - [3, 0, .2, 3, 5, 14, 60], - [1, 0, .4, 3, 6, 15, 77]])) - -mat_y = np.transpose(np.array([[3, 5, 2, 1, 7, 16, 80]])) - -def check_dim(a, b, c): - ''' - Check dimensions - ''' - try: - assert len(a.shape) == 2 & len(b.shape) == 2 & len(c.shape) == 2 - except: - raise Exception('Incorrect data dimension input') - - try: - assert a.shape[0] == b.shape[0] == c.shape[0] - except: - raise Exception('Number of observations not equal') - - try: - assert a.shape[1] <= b.shape[1] - except: - raise Exception('Model underidentified') - -def projection_matrix(b): - ''' - Inputs: - a (matrix) -- x - b (matrix) -- z - ''' - - P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) - return P_b - -def estimate_beta_iv(a, b, c, nocons=""): - ''' - Inputs: - a (matrix) -- x - b (matrix) -- z - c (matrix) -- y - ''' - if nocons != "" & nocons != "nocons": - raise Exception('nocons option misspecified') - - check_dim(a, b, c) - if nocons == "": - N = a.shape[0] - a_1 = np.ones((N,1)) - a = np.hstack((a_1,a)) - b_1 = np.ones((N,1)) - b = np.hstack((b_1, b)) - - - proj = projection_matrix(b) - b_1 = np.transpose(a) @ proj @ a - b_2 = np.linalg.inv(b_1) - b_3 = np.transpose(a) @ proj @ c - betas_all = b_2 @ b_3 - - if nocons == "": - cons = betas_all[0] - betas = betas_all[1:] - print("_cons: " + str(cons)) - for i in range(0, len(betas)): - print("beta_"+ str(i) + " :" + str(betas[i])) - else nocons == "nocons": - betas = betas_all[0:] - for i in range(0, len(betas)): - print("beta_"+ str(i) + " :" + str(betas[i])) - return betas_all - -estimate_beta_iv(mat_x, mat_z, mat_y) - - diff --git a/Projects/project_2_packages/iv_standard_error.py b/Projects/project_2_packages/iv_standard_error.py deleted file mode 100644 index 457c9c4..0000000 --- a/Projects/project_2_packages/iv_standard_error.py +++ /dev/null @@ -1,17 +0,0 @@ - - -import numpy as np - -def calculate_sigma(Z, X, Y, beta_iv): - N = Z.shape[0] - resid = Y - X @ beta_iv - - sigma_iv = (1/N) * np.transpose(resid) @ resid - - return sigma_iv - - - -def calculate_var_beta(sigma, X, Z): - var_beta = sigma^2 * np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) - return var_beta \ No newline at end of file From 914cae78c365be4ba2323a8d51c66454e25d5bfa Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 23:12:54 -0500 Subject: [PATCH 20/43] reformulated no constant argument to use T and F --- Projects/project_2_packages/iv_jett/iv_init.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/iv_init.py b/Projects/project_2_packages/iv_jett/iv_init.py index bdbb3ab..c3d7e5b 100644 --- a/Projects/project_2_packages/iv_jett/iv_init.py +++ b/Projects/project_2_packages/iv_jett/iv_init.py @@ -37,18 +37,18 @@ def projection_matrix(b): P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) return P_b -def estimate_beta_iv(a, b, c, nocons=""): +def estimate_beta_iv(a, b, c, nocons = False): ''' Inputs: a (matrix) -- x b (matrix) -- z c (matrix) -- y ''' - if nocons != "" and nocons != "nocons": + if nocons != True and nocons != False: raise Exception('nocons option misspecified') check_dim(a, b, c) - if nocons == "": + if nocons == False: N = a.shape[0] a_1 = np.ones((N,1)) a = np.hstack((a_1,a)) @@ -62,13 +62,13 @@ def estimate_beta_iv(a, b, c, nocons=""): b_3 = np.transpose(a) @ proj @ c betas_all = b_2 @ b_3 - if nocons == "": + if nocons == False: cons = betas_all[0] betas = betas_all[1:] print("_cons: " + str(cons)) for i in range(0, len(betas)): print("beta_"+ str(i) + " :" + str(betas[i])) - elif nocons == "nocons": + elif nocons == True: betas = betas_all[0:] for i in range(0, len(betas)): print("beta_"+ str(i) + " :" + str(betas[i])) From d504c008612006dc2ddf12d86f1bac10205b0ec7 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 23:16:58 -0500 Subject: [PATCH 21/43] added constant option for sigma calculations --- Projects/project_2_packages/iv_jett/iv_standard_error.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index 7e263b5..1b9259b 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -5,8 +5,14 @@ import numpy as np from iv_jett.iv_init import projection_matrix -def calculate_sigma(Z, X, Y, beta_iv): +def calculate_sigma(Z, X, Y, beta_iv, nocons = False): + N = Z.shape[0] + + if nocons == False: + X_constant = np.ones((N, 1)) + X = np.hstack((X_constant, X)) + resid = Y - X @ beta_iv sigma_iv = (1/N) * np.transpose(resid) @ resid return np.sqrt(sigma_iv) From bd3221f60e32288a2c15e4e1df2aabd793e50117 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 23:23:56 -0500 Subject: [PATCH 22/43] standard errors no account for constant option --- .../project_2_packages/iv_jett/iv_standard_error.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index 1b9259b..381fb59 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -17,7 +17,16 @@ def calculate_sigma(Z, X, Y, beta_iv, nocons = False): sigma_iv = (1/N) * np.transpose(resid) @ resid return np.sqrt(sigma_iv) -def calculate_var_beta(sigma, X, Z): +def calculate_var_beta(sigma, X, Z, nocons = False): + + N = Z.shape[0] + + if nocons == False: + X_constant = np.ones((N, 1)) + X = np.hstack((X_constant, X)) + Z_constant = np.ones((N, 1)) + Z = np.hstack((Z_constant, Z)) + var_beta = (np.asscalar(sigma**2) * np.identity(X.shape[1])) @ np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) se_beta = np.sqrt(np.diag(var_beta)) return se_beta From fe954fa7812bdb6a289e943d37154a0cd7146a73 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Fri, 6 Dec 2019 23:24:14 -0500 Subject: [PATCH 23/43] added printing of steps for easier debugging --- Projects/project_2_packages/data_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Projects/project_2_packages/data_test.py b/Projects/project_2_packages/data_test.py index 317212b..c2e35e9 100644 --- a/Projects/project_2_packages/data_test.py +++ b/Projects/project_2_packages/data_test.py @@ -20,9 +20,9 @@ sigma = calculate_sigma(mat_z, mat_x, mat_y, beta_iv_hat) - +print("sigma") print(sigma) - +print("var beta") var_beta = calculate_var_beta(sigma, mat_x, mat_z) print(var_beta) \ No newline at end of file From 1a3339508e6f5735ebc4b0402347b69eef14c953 Mon Sep 17 00:00:00 2001 From: Nicolaj Thor Date: Sat, 7 Dec 2019 10:01:38 -0500 Subject: [PATCH 24/43] Logit first commit --- Projects/project_2_packages/logit/main.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 Projects/project_2_packages/logit/main.py diff --git a/Projects/project_2_packages/logit/main.py b/Projects/project_2_packages/logit/main.py new file mode 100644 index 0000000..f40e57e --- /dev/null +++ b/Projects/project_2_packages/logit/main.py @@ -0,0 +1,19 @@ +import numpy as np +import math +from scipy.optimize import minimize + +x = np.transpose(np.matrix([0, 1, 3, 0.5, 3, -1, 2])) +y = np.transpose(np.matrix([9, 4, 23, 4, 4, -2, 1])) +def ll_function (x, y, beta): + beta_prime = np.transpose(beta) + fun_1 = y @ beta_prime @ x + fun_2 = math.log(1 + math.exp(beta_prime @ x)) + fun = np.subtract(fun_1, fun_2) + fun = fun.sum() + return fun + +beta0 = np.random.rand(len(x), 1) +print(beta0) +ll_function(x, y, beta0) +res = minimize(ll_function(x, y, beta0), beta0, method='nelder-mead', + options={'xtol': 1e-8, 'disp': True}) From 3c5848e76a743e340b10b42a750d3c8a87363d71 Mon Sep 17 00:00:00 2001 From: Nicolaj Thor Date: Sat, 7 Dec 2019 10:53:37 -0500 Subject: [PATCH 25/43] First tests for IV --- .../build/lib/iv_jett/__init__.py | 0 .../build/lib/iv_jett/iv_init.py | 74 ++++++++++++++++++ .../build/lib/iv_jett/iv_standard_error.py | 32 ++++++++ .../build/lib/iv_jett/test/__init__.py | 0 .../build/lib/iv_jett/test/test_estimates.py | 14 ++++ .../dist/iv_reg_dnu-0.0.1-py3-none-any.whl | Bin 0 -> 4132 bytes .../project_2_packages/iv_jett/iv_init.py | 1 - .../iv_jett/test/__init__.py | 0 .../iv_jett/test/test_estimates.py | 14 ++++ .../iv_reg_dnu.egg-info/PKG-INFO | 14 ++++ .../iv_reg_dnu.egg-info/SOURCES.txt | 10 +++ .../iv_reg_dnu.egg-info/dependency_links.txt | 1 + .../iv_reg_dnu.egg-info/top_level.txt | 1 + Projects/project_2_packages/setup.py | 2 +- 14 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 Projects/project_2_packages/build/lib/iv_jett/__init__.py create mode 100644 Projects/project_2_packages/build/lib/iv_jett/iv_init.py create mode 100644 Projects/project_2_packages/build/lib/iv_jett/iv_standard_error.py create mode 100644 Projects/project_2_packages/build/lib/iv_jett/test/__init__.py create mode 100644 Projects/project_2_packages/build/lib/iv_jett/test/test_estimates.py create mode 100644 Projects/project_2_packages/dist/iv_reg_dnu-0.0.1-py3-none-any.whl create mode 100644 Projects/project_2_packages/iv_jett/test/__init__.py create mode 100644 Projects/project_2_packages/iv_jett/test/test_estimates.py create mode 100644 Projects/project_2_packages/iv_reg_dnu.egg-info/PKG-INFO create mode 100644 Projects/project_2_packages/iv_reg_dnu.egg-info/SOURCES.txt create mode 100644 Projects/project_2_packages/iv_reg_dnu.egg-info/dependency_links.txt create mode 100644 Projects/project_2_packages/iv_reg_dnu.egg-info/top_level.txt diff --git a/Projects/project_2_packages/build/lib/iv_jett/__init__.py b/Projects/project_2_packages/build/lib/iv_jett/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Projects/project_2_packages/build/lib/iv_jett/iv_init.py b/Projects/project_2_packages/build/lib/iv_jett/iv_init.py new file mode 100644 index 0000000..0b7e727 --- /dev/null +++ b/Projects/project_2_packages/build/lib/iv_jett/iv_init.py @@ -0,0 +1,74 @@ +''' + +Python Package to implement instrumental variables +Author: Group 2 NYU Pre docs +Date Created: December 6, 2019 + +''' +import numpy as np + + +def check_dim(a, b, c): + ''' + Check dimensions + ''' + try: + assert len(a.shape) == 2 & len(b.shape) == 2 & len(c.shape) == 2 + except: + raise Exception('Incorrect data dimension input') + + try: + assert a.shape[0] == b.shape[0] == c.shape[0] + except: + raise Exception('Number of observations not equal') + + try: + assert a.shape[1] <= b.shape[1] + except: + raise Exception('Model underidentified') + +def projection_matrix(b): + ''' + Inputs: + b (matrix) -- z + ''' + + P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) + return P_b + +def estimate_beta_iv(a, b, c, nocons = False): + ''' + Inputs: + a (matrix) -- x + b (matrix) -- z + c (matrix) -- y + ''' + if nocons != True and nocons != False: + raise Exception('nocons option misspecified') + + check_dim(a, b, c) + if nocons == False: + N = a.shape[0] + a_1 = np.ones((N,1)) + a = np.hstack((a_1,a)) + b_1 = np.ones((N,1)) + b = np.hstack((b_1, b)) + + + proj = projection_matrix(b) + b_1 = np.transpose(a) @ proj @ a + b_2 = np.linalg.inv(b_1) + b_3 = np.transpose(a) @ proj @ c + betas_all = b_2 @ b_3 + + if nocons == False: + cons = betas_all[0] + betas = betas_all[1:] + print("_cons: " + str(cons)) + for i in range(0, len(betas)): + print("beta_"+ str(i) + " :" + str(betas[i])) + elif nocons == True: + betas = betas_all[0:] + for i in range(0, len(betas)): + print("beta_"+ str(i) + " :" + str(betas[i])) + return betas_all diff --git a/Projects/project_2_packages/build/lib/iv_jett/iv_standard_error.py b/Projects/project_2_packages/build/lib/iv_jett/iv_standard_error.py new file mode 100644 index 0000000..381fb59 --- /dev/null +++ b/Projects/project_2_packages/build/lib/iv_jett/iv_standard_error.py @@ -0,0 +1,32 @@ +''' +Calculate Covariance Matrix +''' + +import numpy as np +from iv_jett.iv_init import projection_matrix + +def calculate_sigma(Z, X, Y, beta_iv, nocons = False): + + N = Z.shape[0] + + if nocons == False: + X_constant = np.ones((N, 1)) + X = np.hstack((X_constant, X)) + + resid = Y - X @ beta_iv + sigma_iv = (1/N) * np.transpose(resid) @ resid + return np.sqrt(sigma_iv) + +def calculate_var_beta(sigma, X, Z, nocons = False): + + N = Z.shape[0] + + if nocons == False: + X_constant = np.ones((N, 1)) + X = np.hstack((X_constant, X)) + Z_constant = np.ones((N, 1)) + Z = np.hstack((Z_constant, Z)) + + var_beta = (np.asscalar(sigma**2) * np.identity(X.shape[1])) @ np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) + se_beta = np.sqrt(np.diag(var_beta)) + return se_beta diff --git a/Projects/project_2_packages/build/lib/iv_jett/test/__init__.py b/Projects/project_2_packages/build/lib/iv_jett/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Projects/project_2_packages/build/lib/iv_jett/test/test_estimates.py b/Projects/project_2_packages/build/lib/iv_jett/test/test_estimates.py new file mode 100644 index 0000000..2d5329e --- /dev/null +++ b/Projects/project_2_packages/build/lib/iv_jett/test/test_estimates.py @@ -0,0 +1,14 @@ +import unittest +import numpy as np +import iv_reg as iv + +class TestBetas(unittest.TestCase): + def test_square_instruments(self): + z = np.random.rand(5, 5) + x = np.random.rand(1, 20) + y = np.random.rand(1, 20) + betas = np.linalg.inv(np.transpose(z) @ x) @ np.transpose(z) @ y + self.assertEqual(betas, iv.estimate_beta_iv(x, z, y, nocons = True)) + +if __name__ == "__main__": + unittest.main() diff --git a/Projects/project_2_packages/dist/iv_reg_dnu-0.0.1-py3-none-any.whl b/Projects/project_2_packages/dist/iv_reg_dnu-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..d9cf1093d43e6a8a4af537ecb9c7432a1b45b260 GIT binary patch literal 4132 zcmai%2|UyPAIImIYmUm1T#-B5T*E>}(aaI%N|GbPge{h%NUj{IB%~a<%aGhhq&X&6 z&LJV!C`TfsU&(*{evkgtzhD3F9-n=`dp!1fK0bSY-g|vsA5%jHMjikFzydJ!k2Ik@ z0O-HJ(JpSx*TcJoqx|Fm8Owq1_1W$lF+2z zH8e$q1moCkh||@j|ezzs!77 z+p%{ylaHty{emp3-HzC zgin`>3)7p}$8zwnmf{CbV2m|kghdyzVad@MmN%0s+qfBR-s>RO(py7+STVjHXd=k!t@mQhAEv)zXg-z9%Lrl{Avt`*7n}++O zXvEtMs}%s^Vr+od=IN_*IF!g7{ZL=BQ3_gK_!Yw3nd-OpbT2%({>=KS^T(q_tY`_c$$a(47~w#RsTH=CW;2a^OA7EI4BN~% z?{9;lKAsMzn$A9XIcWz!+a|`Gd%g&)Ndn=DGC;+QXXG@UBE~+Rtvl*5l|TB8CDcig z?n+`JQplRKf+%0UFDcHF)5k#m6n8b!X-nsYO?a-}_!_LSo$1y2>f;oj!T|c#D<41? zlp5^Kk^`(=t4r%ELt~v{h%1r;wb^Fru814ry`kfI5n=gXFLJxG-`Gl`)C>oAb;xf> ztNAXGp`-k{C5yw)xc0vqJso>3MfS$4Ma3M5U}h>o8hwpCCpa&Te3Ted_4F7q zU#PTB`+gm5aYcUBeHYu)2G-M&_DEaWh&YmalLGs-<&e6tw{3LXwL0=)&xeNv*PwM) zvZ1Hiz|s-{e~m1Sa9&F0RQ+oUReELP_`yT5jf(N)(uo4m?1OxGHlzavr zHbUwT*}hn8==A2-sN|Qey-Ofgm0smRx{Dz)zQscKl)ABJOGmpW)NYWer!rB>3llTL zUpg?pe7=;hU=|K?n!MqV5u{aXWI&S>zYz;xdsFf+64#N zSlL$OmWJ*bo=-6%_P<=Tljz*H>g)n4h-e92i#)sfHHbf-SeiCUp9MFG>VZAcS-T0m z@Nwbg*f==F!kcNQ(4Dk`rEFB4i=w&u2}S@w@}GivV_fZ>y)Hr&pbAh$s591w0Ks~> z;N*=^$MsH{>p_im?a*Vm5rsX$w-UA|o6Y6)#s;gFx)>P-S0!5$GKpMbip}OxW{d3W z4R7o+meg5UKfk|X?}v&AZ62ugQ)h1!t(+So3E6V_tsm7yT-n;#0;bQ61f1gV)ohvu z-Qc|hKQI@&M1s4mXM8Hg3lYZ`q(YlARE8M_JXM4)GL9dkG&I_9ankuN%k`&xW8oc; z+0O)@x1G7LF(FUMxKJC?e#AC~jWx4m-M_ECSd&%6=%MiFXRD^dIx&||cMP%%pzKCm z1Az;qmd(MNB1}e#$@ldSDFlt(;HdFG?>Fo0wU6;ZLH1R^ zs2{mvFYH?Nr|j0O?5XnJsQADV2KZQ}Vvd4aP|TM}oWP+^uBV9&MQvu(zLEYpFq5z_ zITLAiG(_O~`4m|2!iagqELY{iVniRM62~6ljd+V^OQDQ^>oreb$i2MK`goA6`V2?#A_#!_MSNwG#OL6F$hOA&$M4Bgl2Dvk;m2l8oJji> z&PhIV-Rlu?dQK1s7qiLpjgt?-Xl%-s>M?$`77MUB25r*Lp*cFA9-nkd#0x70q(jm) z5`xZ*v!rUrs6k1FPKGtHLz~V%w^=H#H#*2#oG9!#Lmw83lAJPbl;9DxnUO>?fM&XM zD-&dz%+vFk6Iuj5NC#@yxS)=mc6_eNuN|)!_0dVX#l6TOO&WBw

hzk^Z58Z#ITo zs*i;ajFHpI^v)sq!~9fOUaLG5kX8EHVlZ<+GWw$tD)0oq1<`NODnXN8N)M%B)2`VP zTW%gc!wsTWzLJ|6)by}yf~8(fGT1d7=*k+O$e50PW;|`VP5*N%f0`N01=0TZA+#a> z=Tv(VAivCuJ&vo)a^K*XDma^RIzU-sxo1JMiIq8SznuVvW0$lW9EMy$8!G_=k95!_X3G|CPOu z?WU3;N633;`8l8`La#TI4;hx#t|k%UmU-Db#Vj$e1VvV}-0C6rD0gq4!$r%hGFQ8o zwFts@OwSabRN~UI!XL&}$7(Sr6K(tB4e5i3xcM)0r&YvD!Oq)`5b8=ED0L|o; zUY!e%UdmSVhIj1=nicMtzf{Q?>h#wH8VuFgexES%+J$biwHRGSl86h7i{+T|DF9q% z;Zd6=edTgq7i3wYm(`YC2mV~kbBXmA9ceZEK1BaqOKW{SJtI)JzHwJChau$f%g*jH zw8DV$1l8N5%MfA!Zj&!X>l?Izq0#cpvgjU#9%a=*<^CBeTYI;B?>cypLk42d)n-(R zmXSSdOr^4+jH=Kwk6?pc9o>c$J*iJR~GG{xt1S!YMYHVEf(64`SbJ$IJ~_F z#uwuOCHNCMz}LGCAhMIYdEZ^IJ~I;ANoyMsS{@MlGY_qI+yt##)cv$HOPdGS^Ysb{ zA8?q7TXYSmr#o)Y7on08n61X(Oe7*#ZEoVZdf!s* zMA2S;Nw(fTD6z<)TmyCZMDKDc2fi%21|)NmuiB3&j*5uqBb9qDY;Z`ncDP8w#&SCE zOc9>-pHz2C--|(B_N=5@Mqewb=}O?Wk_kAprHmTcYIWUq>%Mxc3A37vb7>eN16%#v z4g{c_#%s^j@#%9h$vjoQu6sMbyCvfRpQCnk)~3blH5M~UKe$OrC=8EuJA+yO)b?eiR;qE0Lx9TZA z@mk5Dg!OW*(SzY_zEv$$;1{4|aGzbq$m0`c_6>BVhII5ifPX(5(fZ-U(e0F@!iy8t`qrrrAArPCe#Hf^W=F#+2Bf3oUs((hu_4k?4S-~S-}8me|v zeiy2CD8s)}{wHGXX8kU2?66pw{+`kQPanS`fA#$xj$-*jtlh1IsW5criMUTxd8w++G~y0R015| GKm7;X@}YD9 literal 0 HcmV?d00001 diff --git a/Projects/project_2_packages/iv_jett/iv_init.py b/Projects/project_2_packages/iv_jett/iv_init.py index c3d7e5b..0b7e727 100644 --- a/Projects/project_2_packages/iv_jett/iv_init.py +++ b/Projects/project_2_packages/iv_jett/iv_init.py @@ -30,7 +30,6 @@ def check_dim(a, b, c): def projection_matrix(b): ''' Inputs: - a (matrix) -- x b (matrix) -- z ''' diff --git a/Projects/project_2_packages/iv_jett/test/__init__.py b/Projects/project_2_packages/iv_jett/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Projects/project_2_packages/iv_jett/test/test_estimates.py b/Projects/project_2_packages/iv_jett/test/test_estimates.py new file mode 100644 index 0000000..12516bc --- /dev/null +++ b/Projects/project_2_packages/iv_jett/test/test_estimates.py @@ -0,0 +1,14 @@ +import unittest +import numpy as np +from iv_jett import iv_init + +class TestBetas(unittest.TestCase): + def test_square_instruments(self): + z = np.random.rand(1, 5) + x = np.random.rand(1, 5) + y = np.random.rand(1, 5) + betas = np.linalg.inv(np.transpose(z) @ x) @ np.transpose(z) @ y + self.assertTrue(np.testing.assert_array_equal(betas, iv_init.estimate_beta_iv(x, z, y, nocons = True))) + +if __name__ == "__main__": + unittest.main() diff --git a/Projects/project_2_packages/iv_reg_dnu.egg-info/PKG-INFO b/Projects/project_2_packages/iv_reg_dnu.egg-info/PKG-INFO new file mode 100644 index 0000000..39f5b71 --- /dev/null +++ b/Projects/project_2_packages/iv_reg_dnu.egg-info/PKG-INFO @@ -0,0 +1,14 @@ +Metadata-Version: 2.1 +Name: iv-reg-dnu +Version: 0.0.1 +Summary: Package for Instrumental Variable Estimation +Home-page: https://github.com/jettpettus/modularizationandtesting +Author: Best Group 2 +License: UNKNOWN +Description: # iv-reg package + + This is a simple package to perform instrumental variable estimation. + +Platform: UNKNOWN +Requires-Python: >=3.6 +Description-Content-Type: text/markdown diff --git a/Projects/project_2_packages/iv_reg_dnu.egg-info/SOURCES.txt b/Projects/project_2_packages/iv_reg_dnu.egg-info/SOURCES.txt new file mode 100644 index 0000000..8870933 --- /dev/null +++ b/Projects/project_2_packages/iv_reg_dnu.egg-info/SOURCES.txt @@ -0,0 +1,10 @@ +setup.py +iv_jett/__init__.py +iv_jett/iv_init.py +iv_jett/iv_standard_error.py +iv_jett/test/__init__.py +iv_jett/test/test_estimates.py +iv_reg_dnu.egg-info/PKG-INFO +iv_reg_dnu.egg-info/SOURCES.txt +iv_reg_dnu.egg-info/dependency_links.txt +iv_reg_dnu.egg-info/top_level.txt \ No newline at end of file diff --git a/Projects/project_2_packages/iv_reg_dnu.egg-info/dependency_links.txt b/Projects/project_2_packages/iv_reg_dnu.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Projects/project_2_packages/iv_reg_dnu.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/Projects/project_2_packages/iv_reg_dnu.egg-info/top_level.txt b/Projects/project_2_packages/iv_reg_dnu.egg-info/top_level.txt new file mode 100644 index 0000000..a5d75b1 --- /dev/null +++ b/Projects/project_2_packages/iv_reg_dnu.egg-info/top_level.txt @@ -0,0 +1 @@ +iv_jett diff --git a/Projects/project_2_packages/setup.py b/Projects/project_2_packages/setup.py index 2703e37..6ebe60e 100644 --- a/Projects/project_2_packages/setup.py +++ b/Projects/project_2_packages/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - name="iv-reg", # Replace with your own username + name="iv_reg_dnu", # Replace with your own username version="0.0.1", author="Best Group 2", description="Package for Instrumental Variable Estimation", From 164b1f1847c5edcc59c3e55face2a0d6914df31d Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Sat, 7 Dec 2019 11:22:09 -0500 Subject: [PATCH 26/43] Pretty --- Projects/project_2_packages/data_test.py | 14 +++++---- .../project_2_packages/iv_jett/iv_init.py | 14 +-------- .../iv_jett/iv_standard_error.py | 30 +++++++++++++++---- Projects/project_2_packages/iv_jett/print.py | 26 ++++++++++++++++ Projects/project_2_packages/iv_jett/run.py | 17 +++++++++++ 5 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 Projects/project_2_packages/iv_jett/print.py create mode 100644 Projects/project_2_packages/iv_jett/run.py diff --git a/Projects/project_2_packages/data_test.py b/Projects/project_2_packages/data_test.py index c2e35e9..cf49f51 100644 --- a/Projects/project_2_packages/data_test.py +++ b/Projects/project_2_packages/data_test.py @@ -1,6 +1,5 @@ import numpy as np -from iv_jett.iv_standard_error import * -from iv_jett.iv_init import * +from iv_jett.run import * mat_x = np.transpose(np.matrix([ [4, .2, 1, 5, 1, 10, 20], [.4, 2, 2, 2, 2, 11, 25], @@ -13,10 +12,9 @@ mat_y = np.transpose(np.array([[3, 5, 2, 1, 7, 16, 80]])) - +''' beta_iv_hat = estimate_beta_iv(mat_x, mat_z, mat_y) - -print(beta_iv_hat) +#print(beta_iv_hat) sigma = calculate_sigma(mat_z, mat_x, mat_y, beta_iv_hat) @@ -25,4 +23,8 @@ print("var beta") var_beta = calculate_var_beta(sigma, mat_x, mat_z) -print(var_beta) \ No newline at end of file +print(var_beta) +''' +ret = TwoStageLeastSquaresRegress(mat_x, mat_y, mat_z, nocons = False, verbose = True) +#print(ret['Beta IV']) +#print(ret['Standard Error']) diff --git a/Projects/project_2_packages/iv_jett/iv_init.py b/Projects/project_2_packages/iv_jett/iv_init.py index c3d7e5b..48f4c55 100644 --- a/Projects/project_2_packages/iv_jett/iv_init.py +++ b/Projects/project_2_packages/iv_jett/iv_init.py @@ -37,7 +37,7 @@ def projection_matrix(b): P_b = b @ np.linalg.inv((np.transpose(b) @ b)) @ np.transpose(b) return P_b -def estimate_beta_iv(a, b, c, nocons = False): +def estimate_beta_iv(a, b, c, nocons = False, verbose = False): ''' Inputs: a (matrix) -- x @@ -55,21 +55,9 @@ def estimate_beta_iv(a, b, c, nocons = False): b_1 = np.ones((N,1)) b = np.hstack((b_1, b)) - proj = projection_matrix(b) b_1 = np.transpose(a) @ proj @ a b_2 = np.linalg.inv(b_1) b_3 = np.transpose(a) @ proj @ c betas_all = b_2 @ b_3 - - if nocons == False: - cons = betas_all[0] - betas = betas_all[1:] - print("_cons: " + str(cons)) - for i in range(0, len(betas)): - print("beta_"+ str(i) + " :" + str(betas[i])) - elif nocons == True: - betas = betas_all[0:] - for i in range(0, len(betas)): - print("beta_"+ str(i) + " :" + str(betas[i])) return betas_all diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index 381fb59..44d24c0 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -6,21 +6,39 @@ from iv_jett.iv_init import projection_matrix def calculate_sigma(Z, X, Y, beta_iv, nocons = False): - + ''' + Description: + Estimates the variance of the errors. + Inputs: + Z (matrix) -- Instrumental Variables + X (matrix) -- Endogenous Variables + Y (matrix) -- Outcome Variable + beta_iv -- IV results + nocons -- Option for constant (default False) + ''' N = Z.shape[0] - if nocons == False: X_constant = np.ones((N, 1)) X = np.hstack((X_constant, X)) - + resid = Y - X @ beta_iv sigma_iv = (1/N) * np.transpose(resid) @ resid return np.sqrt(sigma_iv) -def calculate_var_beta(sigma, X, Z, nocons = False): - - N = Z.shape[0] +def calculate_var_beta(sigma, Z, X, nocons = False): + ''' + Description: + Calculates the variance of beta, the IV estimator + Inputs: + sigma + Z (matrix) -- Instrumental Variables + X (matrix) -- Endogenous Variables + nocons -- Option for constant (default False) + Ouputs: + se_beta (array) -- Standard error + ''' + N = Z.shape[0] if nocons == False: X_constant = np.ones((N, 1)) X = np.hstack((X_constant, X)) diff --git a/Projects/project_2_packages/iv_jett/print.py b/Projects/project_2_packages/iv_jett/print.py new file mode 100644 index 0000000..d90f927 --- /dev/null +++ b/Projects/project_2_packages/iv_jett/print.py @@ -0,0 +1,26 @@ +''' +Print Option +''' + +def print_res(result_dict, nocons = False): + print("\n\033[4m\tResults\t\t\033[0m\n \033[4m\t\t\t\033[0m \n ") + SUB = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") + if nocons == False: + cons = result_dict['Beta IV'][0] + cons_se = result_dict['Standard Error'][0] + betas = result_dict['Beta IV'][1:] + ses = result_dict['Standard Error'][1:] + print("Cons:\t " + str(round(float(cons), 4))) + print("\033[4m\t(" + str(round(cons_se, 5)) + ")\t\033[0m\n") + for i in range(0, len(betas)): + print("\u03B2"+ str(i).translate(SUB) + " : " + \ + str(round(float(betas[i]), 4))) + print("\033[4m\t(" + str(round(float(ses[i]), 5)) + ")\t\033[0m\n") + elif nocons: + betas = result_dict['Beta IV'][0:] + ses = result_dict['Standard Error'][0:] + for i in range(0, len(betas)): + print("\u03B2"+ str(i).translate(SUB) + " : " + \ + str(round(float(betas[i]), 4))) + print("\t(" + str(round(float(ses[i]), 4)) + ")") + print("\n") diff --git a/Projects/project_2_packages/iv_jett/run.py b/Projects/project_2_packages/iv_jett/run.py new file mode 100644 index 0000000..24da544 --- /dev/null +++ b/Projects/project_2_packages/iv_jett/run.py @@ -0,0 +1,17 @@ +''' +Run Package +''' + +from iv_jett.iv_standard_error import * +from iv_jett.iv_init import * +from iv_jett.print import* + +def TwoStageLeastSquaresRegress(X, Y, Z, nocons = False, verbose = False): + beta_iv_hat = estimate_beta_iv(X, Z, Y, nocons) + sigma = calculate_sigma(Z, X, Y, beta_iv_hat, nocons) + var_beta = calculate_var_beta(sigma, X, Z, nocons) + result_dict = {'Beta IV' : beta_iv_hat, + 'Standard Error': var_beta} + if verbose: + print_res(result_dict, nocons) + return result_dict From ad64699b84221c7a45271aa9212b951e3f425a46 Mon Sep 17 00:00:00 2001 From: Nicolaj Thor Date: Sat, 7 Dec 2019 11:27:49 -0500 Subject: [PATCH 27/43] Add test for square matrices --- .../project_2_packages/iv_jett/test/test_estimates.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/test/test_estimates.py b/Projects/project_2_packages/iv_jett/test/test_estimates.py index 12516bc..734b41f 100644 --- a/Projects/project_2_packages/iv_jett/test/test_estimates.py +++ b/Projects/project_2_packages/iv_jett/test/test_estimates.py @@ -4,11 +4,14 @@ class TestBetas(unittest.TestCase): def test_square_instruments(self): - z = np.random.rand(1, 5) - x = np.random.rand(1, 5) - y = np.random.rand(1, 5) + z = np.random.rand(255, 5) + x = np.random.rand(255, 5) + y = np.random.rand(255, 5) + betas = np.linalg.inv(np.transpose(z) @ x) @ np.transpose(z) @ y - self.assertTrue(np.testing.assert_array_equal(betas, iv_init.estimate_beta_iv(x, z, y, nocons = True))) + x2 = iv_init.estimate_beta_iv(x, z, y, nocons = True) + + self.assertIsNone(np.testing.assert_allclose(betas, x2)) if __name__ == "__main__": unittest.main() From 0f4f37f30d4bd4dc1ec38168132afd259717ff62 Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Sat, 7 Dec 2019 11:34:12 -0500 Subject: [PATCH 28/43] Pretty part 2 --- Projects/project_2_packages/iv_jett/print.py | 21 +++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/print.py b/Projects/project_2_packages/iv_jett/print.py index d90f927..0357d00 100644 --- a/Projects/project_2_packages/iv_jett/print.py +++ b/Projects/project_2_packages/iv_jett/print.py @@ -1,9 +1,22 @@ ''' Print Option ''' +class color: + PURPLE = '\033[95m' + CYAN = '\033[96m' + DARKCYAN = '\033[36m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + END = '\033[0m' def print_res(result_dict, nocons = False): - print("\n\033[4m\tResults\t\t\033[0m\n \033[4m\t\t\t\033[0m \n ") + #print("\n\033[4m\tResults\t\t\033[0m\n \033[4m\t\t\t\033[0m \n ") + print( "\n" + color.UNDERLINE + color.BOLD + color.DARKCYAN + \ + "\tResults\t\t\n" + color.END) SUB = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") if nocons == False: cons = result_dict['Beta IV'][0] @@ -11,11 +24,13 @@ def print_res(result_dict, nocons = False): betas = result_dict['Beta IV'][1:] ses = result_dict['Standard Error'][1:] print("Cons:\t " + str(round(float(cons), 4))) - print("\033[4m\t(" + str(round(cons_se, 5)) + ")\t\033[0m\n") + print("\t(" + str(round(cons_se, 5)) + ")\t") + print("------------------------") for i in range(0, len(betas)): print("\u03B2"+ str(i).translate(SUB) + " : " + \ str(round(float(betas[i]), 4))) - print("\033[4m\t(" + str(round(float(ses[i]), 5)) + ")\t\033[0m\n") + print("\t(" + str(round(float(ses[i]), 5)) + ")\t") + print("------------------------") elif nocons: betas = result_dict['Beta IV'][0:] ses = result_dict['Standard Error'][0:] From 21bcfa58874ec94040fa93e5958ba42176fc088c Mon Sep 17 00:00:00 2001 From: Jett Pettus Date: Sat, 7 Dec 2019 11:35:31 -0500 Subject: [PATCH 29/43] Small edits --- Projects/project_2_packages/iv_jett/print.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Projects/project_2_packages/iv_jett/print.py b/Projects/project_2_packages/iv_jett/print.py index 0357d00..a5bcb00 100644 --- a/Projects/project_2_packages/iv_jett/print.py +++ b/Projects/project_2_packages/iv_jett/print.py @@ -37,5 +37,6 @@ def print_res(result_dict, nocons = False): for i in range(0, len(betas)): print("\u03B2"+ str(i).translate(SUB) + " : " + \ str(round(float(betas[i]), 4))) - print("\t(" + str(round(float(ses[i]), 4)) + ")") + print("\t(" + str(round(float(ses[i]), 5)) + ")\t") + print("------------------------") print("\n") From b9145703c1655455119349d623cdcbf1036472c8 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Sat, 7 Dec 2019 11:45:48 -0500 Subject: [PATCH 30/43] logit initial commit --- Projects/project_2_packages/logit/main.py | 34 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/Projects/project_2_packages/logit/main.py b/Projects/project_2_packages/logit/main.py index f40e57e..e4d1ab4 100644 --- a/Projects/project_2_packages/logit/main.py +++ b/Projects/project_2_packages/logit/main.py @@ -4,6 +4,7 @@ x = np.transpose(np.matrix([0, 1, 3, 0.5, 3, -1, 2])) y = np.transpose(np.matrix([9, 4, 23, 4, 4, -2, 1])) + def ll_function (x, y, beta): beta_prime = np.transpose(beta) fun_1 = y @ beta_prime @ x @@ -12,8 +13,37 @@ def ll_function (x, y, beta): fun = fun.sum() return fun + +def fit_logit(x, y): + beta_init = np.random.rand(len(x), 1) + min_func = (lambda beta: ll_function(x, y, beta)) + res = minimize(min_func, x0 = beta_init) + return res + +min_func = (lambda beta: ll_function(x, y, beta)) + + +minimize(min_func, beta_init[:, 0]) +beta_init = np.random.rand(len(x), 1) + min_func = (lambda beta: ll_function(x, y, beta)) +min_func(beta_init) + + +fit_logit(x, y) +res = minimize((lambda beta: ll_function(x, y, beta)), + x0 = beta0 + ) +ll_function(x, y, beta_init) + beta0 = np.random.rand(len(x), 1) print(beta0) ll_function(x, y, beta0) -res = minimize(ll_function(x, y, beta0), beta0, method='nelder-mead', - options={'xtol': 1e-8, 'disp': True}) +res = fmin_tnc(func = ll_function, + approx_grad = True, + x0 = 1, + args = (x, y)) + + + + + From 51b23bda949b07dce53cf63a9787b96f808b63d4 Mon Sep 17 00:00:00 2001 From: Nicolaj Thor Date: Sat, 7 Dec 2019 11:49:34 -0500 Subject: [PATCH 31/43] Add large data stress test --- .../project_2_packages/iv_jett/test/test_estimates.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Projects/project_2_packages/iv_jett/test/test_estimates.py b/Projects/project_2_packages/iv_jett/test/test_estimates.py index 734b41f..a781005 100644 --- a/Projects/project_2_packages/iv_jett/test/test_estimates.py +++ b/Projects/project_2_packages/iv_jett/test/test_estimates.py @@ -13,5 +13,15 @@ def test_square_instruments(self): self.assertIsNone(np.testing.assert_allclose(betas, x2)) + def test_large_data(self): + z = np.random.rand(10^8, 2) + x = np.random.rand(10^8, 2) + y = np.random.rand(10^8, 2) + + betas = np.linalg.inv(np.transpose(z) @ x) @ np.transpose(z) @ y + x2 = iv_init.estimate_beta_iv(x, z, y, nocons = True) + + self.assertIsNone(np.testing.assert_allclose(betas, x2)) + if __name__ == "__main__": unittest.main() From a2e3832ae6a1af5700c3aa7e9ded6f0571faf056 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Sat, 7 Dec 2019 12:02:41 -0500 Subject: [PATCH 32/43] fixed LOGIT YESSSS --- Projects/project_2_packages/logit/main.py | 32 +++++------------------ 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/Projects/project_2_packages/logit/main.py b/Projects/project_2_packages/logit/main.py index e4d1ab4..6f16225 100644 --- a/Projects/project_2_packages/logit/main.py +++ b/Projects/project_2_packages/logit/main.py @@ -2,10 +2,10 @@ import math from scipy.optimize import minimize -x = np.transpose(np.matrix([0, 1, 3, 0.5, 3, -1, 2])) -y = np.transpose(np.matrix([9, 4, 23, 4, 4, -2, 1])) def ll_function (x, y, beta): + # Reshape beta + beta = beta[:, None] beta_prime = np.transpose(beta) fun_1 = y @ beta_prime @ x fun_2 = math.log(1 + math.exp(beta_prime @ x)) @@ -20,30 +20,12 @@ def fit_logit(x, y): res = minimize(min_func, x0 = beta_init) return res -min_func = (lambda beta: ll_function(x, y, beta)) - - -minimize(min_func, beta_init[:, 0]) -beta_init = np.random.rand(len(x), 1) - min_func = (lambda beta: ll_function(x, y, beta)) -min_func(beta_init) - - -fit_logit(x, y) -res = minimize((lambda beta: ll_function(x, y, beta)), - x0 = beta0 - ) -ll_function(x, y, beta_init) - -beta0 = np.random.rand(len(x), 1) -print(beta0) -ll_function(x, y, beta0) -res = fmin_tnc(func = ll_function, - approx_grad = True, - x0 = 1, - args = (x, y)) - +if __name__ == "__main__": + x = np.transpose(np.matrix([0, 1, 3, 0.5, 3, -1, 2])) + y = np.transpose(np.matrix([9, 4, 23, 4, 4, -2, 1])) + + print(fit_logit(x, y)) From 97413ef0adfcc29a719a7db370a139cac7510106 Mon Sep 17 00:00:00 2001 From: Ed Jee Date: Sat, 7 Dec 2019 13:59:50 -0500 Subject: [PATCH 33/43] added tests --- .github/workflows/tests.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..fa94f4f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,15 @@ +name: CI + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Print Directory + run: echo $(pwd) + - name: Run Tests + run: python3 -m unittest -v -s Projects/project_2_packages \ No newline at end of file From 65e0e3d176174e5c5289b92474493dc364bba4cd Mon Sep 17 00:00:00 2001 From: Ed Jee <34009555+EdJeeOnGitHub@users.noreply.github.com> Date: Sat, 7 Dec 2019 14:03:19 -0500 Subject: [PATCH 34/43] Update tests.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fa94f4f..3faebe2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,4 +12,4 @@ jobs: - name: Print Directory run: echo $(pwd) - name: Run Tests - run: python3 -m unittest -v -s Projects/project_2_packages \ No newline at end of file + run: python3 -m unittest discover -s Projects/project_2_packages From fc1ac771cd0b81f0d735731217d71e5bb609e70e Mon Sep 17 00:00:00 2001 From: Ed Jee <34009555+EdJeeOnGitHub@users.noreply.github.com> Date: Sat, 7 Dec 2019 14:03:53 -0500 Subject: [PATCH 35/43] Update tests.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3faebe2..88505d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,6 @@ name: CI -on: [push] +on: [push, pull_request] jobs: build: From 6875458938e504f40cae185241289f22d202326f Mon Sep 17 00:00:00 2001 From: Ed Jee <34009555+EdJeeOnGitHub@users.noreply.github.com> Date: Sat, 7 Dec 2019 14:15:23 -0500 Subject: [PATCH 36/43] Update tests.yml --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 88505d3..792f0b8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,5 +11,7 @@ jobs: - uses: actions/checkout@v1 - name: Print Directory run: echo $(pwd) + - name: Install numoy because I don't understand dependencies + run: pip install numpy - name: Run Tests run: python3 -m unittest discover -s Projects/project_2_packages From 54a0be43268f0c2aa77ef9ade54862b3cdde7acf Mon Sep 17 00:00:00 2001 From: Ed Jee <34009555+EdJeeOnGitHub@users.noreply.github.com> Date: Sat, 7 Dec 2019 14:24:22 -0500 Subject: [PATCH 37/43] Update tests.yml --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 792f0b8..ecee69a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,8 @@ jobs: - uses: actions/checkout@v1 - name: Print Directory run: echo $(pwd) + - name: Docker + run: docker ps; docker ps -a - name: Install numoy because I don't understand dependencies run: pip install numpy - name: Run Tests From 32e46d55d7fb0618e6f41351ec2d06ecc608252d Mon Sep 17 00:00:00 2001 From: Ed Jee <34009555+EdJeeOnGitHub@users.noreply.github.com> Date: Sat, 7 Dec 2019 14:34:26 -0500 Subject: [PATCH 38/43] Create requirements.txt --- Projects/project_2_packages/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Projects/project_2_packages/requirements.txt diff --git a/Projects/project_2_packages/requirements.txt b/Projects/project_2_packages/requirements.txt new file mode 100644 index 0000000..2a09ceb --- /dev/null +++ b/Projects/project_2_packages/requirements.txt @@ -0,0 +1,2 @@ +numpy==1.15.4 +scipy==1.2.1 From 2f2761397bf58851b24e983e6e453c0d943e7da9 Mon Sep 17 00:00:00 2001 From: Ed Jee <34009555+EdJeeOnGitHub@users.noreply.github.com> Date: Sat, 7 Dec 2019 14:34:47 -0500 Subject: [PATCH 39/43] Create requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2a09ceb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +numpy==1.15.4 +scipy==1.2.1 From 83fef1b55c4a8f3685e2f83f15e06cfb59f03047 Mon Sep 17 00:00:00 2001 From: Nicolaj Thor Date: Sat, 7 Dec 2019 14:40:25 -0500 Subject: [PATCH 40/43] travis file --- .travis.yml | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5365a64 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,58 @@ +# Package Development, Testing, and Model Testing + +This repository contains materials for the NYU predoctoral economics training program. This +material covers the topics of code modularization, Python package development, unit and +integration testing, and model testing. It was prepared and presented by Chase Coleman +(Valorum Data) and Jim Savage (Schmidt Future). + + +## 1. Schedule + + +### Friday, December 6 + + +| Time | Topic | Materials | +| :----: | :--------- | :-------- | +| 9:00 - 12:30 | | | +| | Modularization | | +| | Project 1 | [Project 1](https://github.com/nyupredocs/modularizationandtesting/blob/master/Projects/Project_1_HendricksLeukhina.ipynb) | +| | Package Development | [Day 1 slides](https://github.com/nyupredocs/modularizationandtesting/blob/master/Slides/package_dev_FriAM.pdf) | +| 12:30 - 13:30 | Lunch | | +| 13:30 - 17:00 | Wrap up remaining slides | | +| | Project 2 | [Project 2](https://github.com/nyupredocs/modularizationandtesting/blob/master/Projects/Project_2_PackageBuilding.md) | + + +### Saturday, December 7 + +| Time | Topic | Materials | +| :----: | :--------- | :-------- | + + +### Sunday, December 8 + +| Time | Topic | Materials | +| :----: | :--------- | :-------- | + + +## 2. Package Development + +The following mantra should be more widely adopted: “Don’t write a new package, if there is already a high-quality package that fulfills your needs.” + +Below are some examples of scientific projects that the Python community typically deems as “high-quality.” + +* [Astropy](https://www.astropy.org/about.html#about-the-astropy-project) +* [Biopython](https://biopython.org/) +* [pyblp](https://pyblp.readthedocs.io/en/stable/) +* [pymc](https://docs.pymc.io/) and Stan](https://mc-stan.org/) +* [sunpy](https://sunpy.org/) +* [quantecon](https://quantecon.org/quantecon-py/) + +What do these projects have in common? + +Here are some additional readings: + +* [Instructions for creating Python package](https://packaging.python.org/tutorials/packaging-projects/) +* [How to develop good R packages](https://www.r-bloggers.com/how-to-develop-good-r-packages-for-open-science/) (for open science): Not about Python but does have some thoughtful points on general package development +* [Open Reproducible Science](https://www.earthdatascience.org/courses/intro-to-earth-data-science/open-reproducible-science/get-started-open-reproducible-science/) +* [Advantages of Modularization in Programming](https://www.techwalla.com/articles/the-advantages-of-modularization-in-programming) From 47b9ca91da01a3ed5d0f5a245cd63f3d9ac564af Mon Sep 17 00:00:00 2001 From: Nicolaj Thor Date: Sat, 7 Dec 2019 14:42:31 -0500 Subject: [PATCH 41/43] Fix TRAVIS file --- .travis.yml | 65 ++++++----------------------------------------------- 1 file changed, 7 insertions(+), 58 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5365a64..8fbe456 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,58 +1,7 @@ -# Package Development, Testing, and Model Testing - -This repository contains materials for the NYU predoctoral economics training program. This -material covers the topics of code modularization, Python package development, unit and -integration testing, and model testing. It was prepared and presented by Chase Coleman -(Valorum Data) and Jim Savage (Schmidt Future). - - -## 1. Schedule - - -### Friday, December 6 - - -| Time | Topic | Materials | -| :----: | :--------- | :-------- | -| 9:00 - 12:30 | | | -| | Modularization | | -| | Project 1 | [Project 1](https://github.com/nyupredocs/modularizationandtesting/blob/master/Projects/Project_1_HendricksLeukhina.ipynb) | -| | Package Development | [Day 1 slides](https://github.com/nyupredocs/modularizationandtesting/blob/master/Slides/package_dev_FriAM.pdf) | -| 12:30 - 13:30 | Lunch | | -| 13:30 - 17:00 | Wrap up remaining slides | | -| | Project 2 | [Project 2](https://github.com/nyupredocs/modularizationandtesting/blob/master/Projects/Project_2_PackageBuilding.md) | - - -### Saturday, December 7 - -| Time | Topic | Materials | -| :----: | :--------- | :-------- | - - -### Sunday, December 8 - -| Time | Topic | Materials | -| :----: | :--------- | :-------- | - - -## 2. Package Development - -The following mantra should be more widely adopted: “Don’t write a new package, if there is already a high-quality package that fulfills your needs.” - -Below are some examples of scientific projects that the Python community typically deems as “high-quality.” - -* [Astropy](https://www.astropy.org/about.html#about-the-astropy-project) -* [Biopython](https://biopython.org/) -* [pyblp](https://pyblp.readthedocs.io/en/stable/) -* [pymc](https://docs.pymc.io/) and Stan](https://mc-stan.org/) -* [sunpy](https://sunpy.org/) -* [quantecon](https://quantecon.org/quantecon-py/) - -What do these projects have in common? - -Here are some additional readings: - -* [Instructions for creating Python package](https://packaging.python.org/tutorials/packaging-projects/) -* [How to develop good R packages](https://www.r-bloggers.com/how-to-develop-good-r-packages-for-open-science/) (for open science): Not about Python but does have some thoughtful points on general package development -* [Open Reproducible Science](https://www.earthdatascience.org/courses/intro-to-earth-data-science/open-reproducible-science/get-started-open-reproducible-science/) -* [Advantages of Modularization in Programming](https://www.techwalla.com/articles/the-advantages-of-modularization-in-programming) +language: python +# command to install dependencies +install: + - pip install -r requirements.txt +# command to run tests +script: + - unittest From 9f8212193f6c1e0b3fbc8e04bcdae3d1636f5818 Mon Sep 17 00:00:00 2001 From: Nicolas Jimenez Date: Sat, 7 Dec 2019 14:44:17 -0500 Subject: [PATCH 42/43] robust SEs attempt --- .../iv_jett/iv_standard_error.py | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/Projects/project_2_packages/iv_jett/iv_standard_error.py b/Projects/project_2_packages/iv_jett/iv_standard_error.py index 381fb59..51cf513 100644 --- a/Projects/project_2_packages/iv_jett/iv_standard_error.py +++ b/Projects/project_2_packages/iv_jett/iv_standard_error.py @@ -5,6 +5,14 @@ import numpy as np from iv_jett.iv_init import projection_matrix +def calculate_resid(Z, X, Y, beta_iv, nocons = False): + if nocons == False: + X_constant = np.ones((N, 1)) + X = np.hstack((X_constant, X)) + + resid = Y - X @ beta_iv + return resid + def calculate_sigma(Z, X, Y, beta_iv, nocons = False): N = Z.shape[0] @@ -12,12 +20,13 @@ def calculate_sigma(Z, X, Y, beta_iv, nocons = False): if nocons == False: X_constant = np.ones((N, 1)) X = np.hstack((X_constant, X)) - + resid = Y - X @ beta_iv + sigma_iv = (1/N) * np.transpose(resid) @ resid return np.sqrt(sigma_iv) -def calculate_var_beta(sigma, X, Z, nocons = False): +def calculate_var_beta(sigma, X, Z, resid, nocons = False, robust = False): N = Z.shape[0] @@ -27,6 +36,17 @@ def calculate_var_beta(sigma, X, Z, nocons = False): Z_constant = np.ones((N, 1)) Z = np.hstack((Z_constant, Z)) - var_beta = (np.asscalar(sigma**2) * np.identity(X.shape[1])) @ np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) - se_beta = np.sqrt(np.diag(var_beta)) + if robust == False: + var_beta = (np.asscalar(sigma**2) * np.identity(X.shape[1])) @ np.linalg.inv(np.transpose(X) @ projection_matrix(Z) @ X) + se_beta = np.sqrt(np.diag(var_beta)) + + if robust == True: + resid_sq = resid @ np.transpose(resid) + print("resid sq:") + print(resid_sq) + sandwich_bread = np.linalg.inv(np.transpose(Z) @ X) + var_beta = (sandwich_bread @ np.transpose(Z) @ np.diag(np.diag(resid_sq)) @ Z @ sandwich_bread) + print("var_beta") + print(var_beta) + se_beta = np.sqrt(np.diag(var_beta)) return se_beta From 75a14f90cd6675d482ccfb63f33700cea4f95000 Mon Sep 17 00:00:00 2001 From: Nicolas Jimenez Date: Sat, 7 Dec 2019 15:13:22 -0500 Subject: [PATCH 43/43] added robust se option --- Projects/project_2_packages/iv_jett/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Projects/project_2_packages/iv_jett/run.py b/Projects/project_2_packages/iv_jett/run.py index 24da544..381aaaf 100644 --- a/Projects/project_2_packages/iv_jett/run.py +++ b/Projects/project_2_packages/iv_jett/run.py @@ -8,8 +8,9 @@ def TwoStageLeastSquaresRegress(X, Y, Z, nocons = False, verbose = False): beta_iv_hat = estimate_beta_iv(X, Z, Y, nocons) + resid = calculate_resid(Z, X, Y, beta_iv_hat, nocons) sigma = calculate_sigma(Z, X, Y, beta_iv_hat, nocons) - var_beta = calculate_var_beta(sigma, X, Z, nocons) + var_beta = calculate_var_beta(sigma, X, Z, resid, nocons) result_dict = {'Beta IV' : beta_iv_hat, 'Standard Error': var_beta} if verbose: