head	1.245;
access;
symbols;
locks;
comment	@# @;


1.245
date	2010.01.01.20.17.56;	author debdev;	state Exp;
branches;
next	1.244;

1.244
date	2010.01.01.20.02.32;	author debdev;	state Exp;
branches;
next	1.243;

1.243
date	2009.12.07.16.00.11;	author debdev;	state Exp;
branches;
next	1.242;

1.242
date	2009.12.07.15.51.16;	author debdev;	state Exp;
branches;
next	1.241;

1.241
date	2009.12.06.22.05.32;	author debdev;	state Exp;
branches;
next	1.240;

1.240
date	2009.12.06.21.46.18;	author debdev;	state Exp;
branches;
next	1.239;

1.239
date	2009.12.06.19.02.26;	author debdev;	state Exp;
branches;
next	1.238;

1.238
date	2009.11.15.17.38.38;	author debdev;	state Exp;
branches;
next	1.237;

1.237
date	2009.11.15.17.33.03;	author debdev;	state Exp;
branches;
next	1.236;

1.236
date	2009.11.08.20.18.54;	author debdev;	state Exp;
branches;
next	1.235;

1.235
date	2009.10.29.07.12.46;	author debdev;	state Exp;
branches;
next	1.234;

1.234
date	2009.10.28.20.15.06;	author debdev;	state Exp;
branches;
next	1.233;

1.233
date	2009.10.28.20.13.43;	author debdev;	state Exp;
branches;
next	1.232;

1.232
date	2009.10.28.20.00.33;	author debdev;	state Exp;
branches;
next	1.231;

1.231
date	2009.10.28.19.11.46;	author debdev;	state Exp;
branches;
next	1.230;

1.230
date	2009.10.28.18.31.28;	author debdev;	state Exp;
branches;
next	1.229;

1.229
date	2009.10.28.18.25.36;	author debdev;	state Exp;
branches;
next	1.228;

1.228
date	2009.10.28.18.24.56;	author debdev;	state Exp;
branches;
next	1.227;

1.227
date	2009.10.28.18.23.13;	author debdev;	state Exp;
branches;
next	1.226;

1.226
date	2009.10.25.13.58.54;	author debdev;	state Exp;
branches;
next	1.225;

1.225
date	2009.10.25.13.47.46;	author debdev;	state Exp;
branches;
next	1.224;

1.224
date	2009.10.25.13.39.35;	author debdev;	state Exp;
branches;
next	1.223;

1.223
date	2009.10.25.12.37.31;	author debdev;	state Exp;
branches;
next	1.222;

1.222
date	2009.10.25.11.29.54;	author debdev;	state Exp;
branches;
next	1.221;

1.221
date	2009.10.25.11.27.34;	author debdev;	state Exp;
branches;
next	1.220;

1.220
date	2009.10.24.15.03.01;	author debdev;	state Exp;
branches;
next	1.219;

1.219
date	2009.10.24.13.07.43;	author debdev;	state Exp;
branches;
next	1.218;

1.218
date	2009.10.24.12.58.26;	author debdev;	state Exp;
branches;
next	1.217;

1.217
date	2009.10.24.12.43.20;	author debdev;	state Exp;
branches;
next	1.216;

1.216
date	2009.10.18.14.08.10;	author debdev;	state Exp;
branches;
next	1.215;

1.215
date	2009.09.22.17.35.30;	author debdev;	state Exp;
branches;
next	1.214;

1.214
date	2009.09.22.17.21.06;	author debdev;	state Exp;
branches;
next	1.213;

1.213
date	2009.09.18.13.39.58;	author debdev;	state Exp;
branches;
next	1.212;

1.212
date	2009.09.12.14.29.18;	author debdev;	state Exp;
branches;
next	1.211;

1.211
date	2009.09.11.12.37.07;	author debdev;	state Exp;
branches;
next	1.210;

1.210
date	2009.09.11.12.35.08;	author debdev;	state Exp;
branches;
next	1.209;

1.209
date	2009.09.11.11.46.11;	author debdev;	state Exp;
branches;
next	1.208;

1.208
date	2009.09.07.15.25.10;	author debdev;	state Exp;
branches;
next	1.207;

1.207
date	2009.09.07.15.19.04;	author debdev;	state Exp;
branches;
next	1.206;

1.206
date	2009.09.07.15.16.50;	author debdev;	state Exp;
branches;
next	1.205;

1.205
date	2009.09.05.15.08.17;	author debdev;	state Exp;
branches;
next	1.204;

1.204
date	2009.09.04.22.01.19;	author debdev;	state Exp;
branches;
next	1.203;

1.203
date	2009.09.04.21.56.50;	author debdev;	state Exp;
branches;
next	1.202;

1.202
date	2009.09.04.18.00.43;	author debdev;	state Exp;
branches;
next	1.201;

1.201
date	2009.09.04.15.51.11;	author debdev;	state Exp;
branches;
next	1.200;

1.200
date	2009.09.04.15.18.00;	author debdev;	state Exp;
branches;
next	1.199;

1.199
date	2009.09.04.15.15.52;	author debdev;	state Exp;
branches;
next	1.198;

1.198
date	2009.09.04.13.42.59;	author debdev;	state Exp;
branches;
next	1.197;

1.197
date	2009.09.04.13.28.11;	author debdev;	state Exp;
branches;
next	1.196;

1.196
date	2009.09.03.21.40.17;	author debdev;	state Exp;
branches;
next	1.195;

1.195
date	2009.08.29.06.25.09;	author debdev;	state Exp;
branches;
next	1.194;

1.194
date	2009.08.28.21.27.52;	author debdev;	state Exp;
branches;
next	1.193;

1.193
date	2009.08.28.21.13.49;	author debdev;	state Exp;
branches;
next	1.192;

1.192
date	2009.08.28.21.05.19;	author debdev;	state Exp;
branches;
next	1.191;

1.191
date	2009.08.28.20.53.42;	author debdev;	state Exp;
branches;
next	1.190;

1.190
date	2009.08.28.20.48.00;	author debdev;	state Exp;
branches;
next	1.189;

1.189
date	2009.08.28.20.45.55;	author debdev;	state Exp;
branches;
next	1.188;

1.188
date	2009.08.28.20.14.47;	author debdev;	state Exp;
branches;
next	1.187;

1.187
date	2009.08.28.20.11.49;	author debdev;	state Exp;
branches;
next	1.186;

1.186
date	2009.08.28.20.08.30;	author debdev;	state Exp;
branches;
next	1.185;

1.185
date	2009.08.28.20.00.41;	author debdev;	state Exp;
branches;
next	1.184;

1.184
date	2009.08.28.19.28.00;	author debdev;	state Exp;
branches;
next	1.183;

1.183
date	2009.08.28.19.19.08;	author debdev;	state Exp;
branches;
next	1.182;

1.182
date	2009.08.28.18.26.23;	author debdev;	state Exp;
branches;
next	1.181;

1.181
date	2009.08.28.18.25.31;	author debdev;	state Exp;
branches;
next	1.180;

1.180
date	2009.08.28.17.56.58;	author debdev;	state Exp;
branches;
next	1.179;

1.179
date	2009.08.28.17.55.00;	author debdev;	state Exp;
branches;
next	1.178;

1.178
date	2009.08.28.17.53.00;	author debdev;	state Exp;
branches;
next	1.177;

1.177
date	2009.08.28.17.51.51;	author debdev;	state Exp;
branches;
next	1.176;

1.176
date	2009.08.28.17.47.54;	author debdev;	state Exp;
branches;
next	1.175;

1.175
date	2009.08.28.10.30.39;	author debdev;	state Exp;
branches;
next	1.174;

1.174
date	2009.08.28.09.08.45;	author debdev;	state Exp;
branches;
next	1.173;

1.173
date	2009.08.27.20.51.03;	author debdev;	state Exp;
branches;
next	1.172;

1.172
date	2009.08.27.20.46.41;	author debdev;	state Exp;
branches;
next	1.171;

1.171
date	2009.08.27.20.35.41;	author debdev;	state Exp;
branches;
next	1.170;

1.170
date	2009.08.27.13.18.08;	author debdev;	state Exp;
branches;
next	1.169;

1.169
date	2009.08.27.12.59.13;	author debdev;	state Exp;
branches;
next	1.168;

1.168
date	2009.08.27.12.13.29;	author debdev;	state Exp;
branches;
next	1.167;

1.167
date	2009.08.27.12.05.31;	author debdev;	state Exp;
branches;
next	1.166;

1.166
date	2009.08.27.11.53.00;	author debdev;	state Exp;
branches;
next	1.165;

1.165
date	2009.08.27.11.50.12;	author debdev;	state Exp;
branches;
next	1.164;

1.164
date	2009.08.26.19.29.06;	author debdev;	state Exp;
branches;
next	1.163;

1.163
date	2009.08.26.19.28.04;	author debdev;	state Exp;
branches;
next	1.162;

1.162
date	2009.08.24.09.30.58;	author debdev;	state Exp;
branches;
next	1.161;

1.161
date	2009.08.22.17.32.32;	author debdev;	state Exp;
branches;
next	1.160;

1.160
date	2009.08.22.17.31.15;	author debdev;	state Exp;
branches;
next	1.159;

1.159
date	2009.08.22.17.29.43;	author debdev;	state Exp;
branches;
next	1.158;

1.158
date	2009.08.22.16.59.02;	author debdev;	state Exp;
branches;
next	1.157;

1.157
date	2009.08.22.16.47.39;	author debdev;	state Exp;
branches;
next	1.156;

1.156
date	2009.08.22.12.34.52;	author debdev;	state Exp;
branches;
next	1.155;

1.155
date	2009.08.22.12.25.50;	author debdev;	state Exp;
branches;
next	1.154;

1.154
date	2009.08.22.12.25.32;	author debdev;	state Exp;
branches;
next	1.153;

1.153
date	2009.08.22.12.17.39;	author debdev;	state Exp;
branches;
next	1.152;

1.152
date	2009.08.22.10.57.38;	author debdev;	state Exp;
branches;
next	1.151;

1.151
date	2009.08.22.10.50.17;	author debdev;	state Exp;
branches;
next	1.150;

1.150
date	2009.08.22.09.48.41;	author debdev;	state Exp;
branches;
next	1.149;

1.149
date	2009.08.22.07.02.08;	author debdev;	state Exp;
branches;
next	1.148;

1.148
date	2009.08.21.12.19.07;	author debdev;	state Exp;
branches;
next	1.147;

1.147
date	2009.08.21.10.57.19;	author debdev;	state Exp;
branches;
next	1.146;

1.146
date	2009.08.21.10.18.35;	author debdev;	state Exp;
branches;
next	1.145;

1.145
date	2009.08.21.08.16.41;	author debdev;	state Exp;
branches;
next	1.144;

1.144
date	2009.08.07.08.15.18;	author debdev;	state Exp;
branches;
next	1.143;

1.143
date	2009.08.06.21.02.31;	author debdev;	state Exp;
branches;
next	1.142;

1.142
date	2009.08.05.15.39.04;	author debdev;	state Exp;
branches;
next	1.141;

1.141
date	2009.08.05.12.12.02;	author debdev;	state Exp;
branches;
next	1.140;

1.140
date	2009.08.05.10.35.04;	author debdev;	state Exp;
branches;
next	1.139;

1.139
date	2009.08.05.09.14.42;	author debdev;	state Exp;
branches;
next	1.138;

1.138
date	2009.04.08.12.14.46;	author debdev;	state Exp;
branches;
next	1.137;

1.137
date	2009.04.08.11.46.20;	author debdev;	state Exp;
branches;
next	1.136;

1.136
date	2009.03.29.18.52.47;	author debdev;	state Exp;
branches;
next	1.135;

1.135
date	2008.11.27.12.17.12;	author debdev;	state Exp;
branches;
next	1.134;

1.134
date	2008.05.03.17.19.10;	author debdev;	state Exp;
branches;
next	1.133;

1.133
date	2008.03.16.11.06.07;	author debdev;	state Exp;
branches;
next	1.132;

1.132
date	2008.03.10.13.53.02;	author debdev;	state Exp;
branches;
next	1.131;

1.131
date	2008.03.10.13.45.14;	author debdev;	state Exp;
branches;
next	1.130;

1.130
date	2008.03.10.12.42.50;	author debdev;	state Exp;
branches;
next	1.129;

1.129
date	2007.12.27.16.48.59;	author debdev;	state Exp;
branches;
next	1.128;

1.128
date	2007.12.22.08.01.41;	author debdev;	state Exp;
branches;
next	1.127;

1.127
date	2007.12.21.21.37.00;	author debdev;	state Exp;
branches;
next	1.126;

1.126
date	2007.12.17.14.26.35;	author debdev;	state Exp;
branches;
next	1.125;

1.125
date	2007.11.15.10.18.24;	author debdev;	state Exp;
branches;
next	1.124;

1.124
date	2007.11.15.10.12.53;	author debdev;	state Exp;
branches;
next	1.123;

1.123
date	2007.11.15.10.06.01;	author debdev;	state Exp;
branches;
next	1.122;

1.122
date	2007.11.15.09.14.03;	author debdev;	state Exp;
branches;
next	1.121;

1.121
date	2007.11.13.13.47.44;	author debdev;	state Exp;
branches;
next	1.120;

1.120
date	2007.11.13.13.34.39;	author debdev;	state Exp;
branches;
next	1.119;

1.119
date	2007.11.13.13.33.25;	author debdev;	state Exp;
branches;
next	1.118;

1.118
date	2007.11.08.20.24.53;	author debdev;	state Exp;
branches;
next	1.117;

1.117
date	2007.11.07.10.43.37;	author debdev;	state Exp;
branches;
next	1.116;

1.116
date	2007.10.20.17.14.39;	author debdev;	state Exp;
branches;
next	1.115;

1.115
date	2007.10.17.13.11.46;	author debdev;	state Exp;
branches;
next	1.114;

1.114
date	2007.10.17.13.00.24;	author debdev;	state Exp;
branches;
next	1.113;

1.113
date	2007.08.26.18.20.05;	author debdev;	state Exp;
branches;
next	1.112;

1.112
date	2007.08.26.18.10.48;	author debdev;	state Exp;
branches;
next	1.111;

1.111
date	2007.07.19.12.05.55;	author debdev;	state Exp;
branches;
next	1.110;

1.110
date	2007.07.18.12.20.59;	author debdev;	state Exp;
branches;
next	1.109;

1.109
date	2007.07.17.08.09.07;	author debdev;	state Exp;
branches;
next	1.108;

1.108
date	2007.07.16.07.41.39;	author debdev;	state Exp;
branches;
next	1.107;

1.107
date	2007.04.13.09.57.31;	author debdev;	state Exp;
branches;
next	1.106;

1.106
date	2007.04.13.09.49.03;	author debdev;	state Exp;
branches;
next	1.105;

1.105
date	2006.12.21.13.02.11;	author debdev;	state Exp;
branches;
next	1.104;

1.104
date	2006.09.07.19.02.57;	author debdev;	state Exp;
branches;
next	1.103;

1.103
date	2006.09.07.18.47.01;	author debdev;	state Exp;
branches;
next	1.102;

1.102
date	2006.07.23.09.15.04;	author debdev;	state Exp;
branches;
next	1.101;

1.101
date	2006.07.12.20.14.36;	author debdev;	state Exp;
branches;
next	1.100;

1.100
date	2006.07.12.15.10.13;	author debdev;	state Exp;
branches;
next	1.99;

1.99
date	2006.07.09.17.02.02;	author debdev;	state Exp;
branches;
next	1.98;

1.98
date	2006.07.09.08.42.36;	author debdev;	state Exp;
branches;
next	1.97;

1.97
date	2006.07.09.08.21.46;	author debdev;	state Exp;
branches;
next	1.96;

1.96
date	2006.07.09.07.22.34;	author debdev;	state Exp;
branches;
next	1.95;

1.95
date	2006.07.07.20.42.36;	author debdev;	state Exp;
branches;
next	1.94;

1.94
date	2006.07.07.20.24.25;	author debdev;	state Exp;
branches;
next	1.93;

1.93
date	2006.07.07.20.21.05;	author debdev;	state Exp;
branches;
next	1.92;

1.92
date	2006.07.07.20.12.56;	author debdev;	state Exp;
branches;
next	1.91;

1.91
date	2006.07.07.20.08.16;	author debdev;	state Exp;
branches;
next	1.90;

1.90
date	2006.07.07.20.03.14;	author debdev;	state Exp;
branches;
next	1.89;

1.89
date	2006.07.07.12.41.53;	author debdev;	state Exp;
branches;
next	1.88;

1.88
date	2006.07.07.12.04.27;	author debdev;	state Exp;
branches;
next	1.87;

1.87
date	2006.07.07.12.01.56;	author debdev;	state Exp;
branches;
next	1.86;

1.86
date	2006.07.07.11.59.07;	author debdev;	state Exp;
branches;
next	1.85;

1.85
date	2006.07.07.11.57.01;	author debdev;	state Exp;
branches;
next	1.84;

1.84
date	2006.07.07.11.56.06;	author debdev;	state Exp;
branches;
next	1.83;

1.83
date	2006.07.07.11.55.35;	author debdev;	state Exp;
branches;
next	1.82;

1.82
date	2006.07.06.16.57.36;	author debdev;	state Exp;
branches;
next	1.81;

1.81
date	2006.07.06.16.52.03;	author debdev;	state Exp;
branches;
next	1.80;

1.80
date	2006.06.30.14.20.40;	author debdev;	state Exp;
branches;
next	1.79;

1.79
date	2006.06.30.14.13.12;	author debdev;	state Exp;
branches;
next	1.78;

1.78
date	2006.06.29.08.07.13;	author debdev;	state Exp;
branches;
next	1.77;

1.77
date	2006.06.27.20.21.22;	author debdev;	state Exp;
branches;
next	1.76;

1.76
date	2006.06.23.15.04.52;	author debdev;	state Exp;
branches;
next	1.75;

1.75
date	2006.06.23.14.52.50;	author debdev;	state Exp;
branches;
next	1.74;

1.74
date	2006.06.23.11.26.14;	author debdev;	state Exp;
branches;
next	1.73;

1.73
date	2006.06.23.10.02.43;	author debdev;	state Exp;
branches;
next	1.72;

1.72
date	2006.06.23.09.57.09;	author debdev;	state Exp;
branches;
next	1.71;

1.71
date	2006.06.23.09.39.04;	author debdev;	state Exp;
branches;
next	1.70;

1.70
date	2006.06.23.09.27.47;	author debdev;	state Exp;
branches;
next	1.69;

1.69
date	2006.06.23.09.17.06;	author debdev;	state Exp;
branches;
next	1.68;

1.68
date	2006.06.23.08.16.35;	author debdev;	state Exp;
branches;
next	1.67;

1.67
date	2006.06.23.07.27.20;	author debdev;	state Exp;
branches;
next	1.66;

1.66
date	2006.06.22.16.00.34;	author debdev;	state Exp;
branches;
next	1.65;

1.65
date	2006.06.22.12.11.14;	author mennucci;	state Exp;
branches;
next	1.64;

1.64
date	2006.06.21.10.12.53;	author mennucci;	state Exp;
branches;
next	1.63;

1.63
date	2006.06.20.10.11.26;	author mennucci;	state Exp;
branches;
next	1.62;

1.62
date	2006.06.20.06.08.12;	author debdev;	state Exp;
branches;
next	1.61;

1.61
date	2006.06.20.06.05.28;	author debdev;	state Exp;
branches;
next	1.60;

1.60
date	2006.06.19.21.11.28;	author debdev;	state Exp;
branches;
next	1.59;

1.59
date	2006.06.19.12.45.01;	author mennucci;	state Exp;
branches;
next	1.58;

1.58
date	2006.06.19.10.13.09;	author mennucci;	state Exp;
branches;
next	1.57;

1.57
date	2006.06.19.09.03.51;	author mennucci;	state Exp;
branches;
next	1.56;

1.56
date	2006.06.19.08.46.14;	author mennucci;	state Exp;
branches;
next	1.55;

1.55
date	2006.06.18.17.26.04;	author debdev;	state Exp;
branches;
next	1.54;

1.54
date	2006.06.18.16.08.31;	author debdev;	state Exp;
branches;
next	1.53;

1.53
date	2006.06.16.16.46.54;	author debdev;	state Exp;
branches;
next	1.52;

1.52
date	2006.06.16.16.37.51;	author debdev;	state Exp;
branches;
next	1.51;

1.51
date	2006.06.16.16.26.59;	author debdev;	state Exp;
branches;
next	1.50;

1.50
date	2006.06.15.19.35.04;	author debdev;	state Exp;
branches;
next	1.49;

1.49
date	2006.06.15.11.52.55;	author debdev;	state Exp;
branches;
next	1.48;

1.48
date	2006.06.15.11.25.13;	author debdev;	state Exp;
branches;
next	1.47;

1.47
date	2006.06.15.10.39.39;	author debdev;	state Exp;
branches;
next	1.46;

1.46
date	2006.06.15.10.12.07;	author debdev;	state Exp;
branches;
next	1.45;

1.45
date	2006.06.15.07.16.25;	author debdev;	state Exp;
branches;
next	1.44;

1.44
date	2006.06.15.06.52.23;	author debdev;	state Exp;
branches;
next	1.43;

1.43
date	2006.06.14.08.00.51;	author mennucci;	state Exp;
branches;
next	1.42;

1.42
date	2006.06.13.10.14.50;	author debdev;	state Exp;
branches;
next	1.41;

1.41
date	2006.06.13.07.54.39;	author debdev;	state Exp;
branches;
next	1.40;

1.40
date	2006.06.13.07.39.23;	author debdev;	state Exp;
branches;
next	1.39;

1.39
date	2006.06.13.07.27.38;	author debdev;	state Exp;
branches;
next	1.38;

1.38
date	2006.06.13.07.08.30;	author debdev;	state Exp;
branches;
next	1.37;

1.37
date	2006.06.12.17.56.32;	author debdev;	state Exp;
branches;
next	1.36;

1.36
date	2006.06.12.17.41.33;	author debdev;	state Exp;
branches;
next	1.35;

1.35
date	2006.06.12.09.53.29;	author debdev;	state Exp;
branches;
next	1.34;

1.34
date	2006.06.12.09.25.21;	author debdev;	state Exp;
branches;
next	1.33;

1.33
date	2006.06.11.19.01.14;	author debdev;	state Exp;
branches;
next	1.32;

1.32
date	2006.06.11.16.13.50;	author debdev;	state Exp;
branches;
next	1.31;

1.31
date	2006.06.11.10.16.20;	author debdev;	state Exp;
branches;
next	1.30;

1.30
date	2006.06.11.09.18.16;	author debdev;	state Exp;
branches;
next	1.29;

1.29
date	2006.06.11.06.42.36;	author debdev;	state Exp;
branches;
next	1.28;

1.28
date	2006.06.10.11.12.20;	author debdev;	state Exp;
branches;
next	1.27;

1.27
date	2006.06.07.14.05.18;	author debdev;	state Exp;
branches;
next	1.26;

1.26
date	2006.06.01.21.10.06;	author debdev;	state Exp;
branches;
next	1.25;

1.25
date	2006.05.31.14.14.16;	author debdev;	state Exp;
branches;
next	1.24;

1.24
date	2006.05.31.13.13.39;	author debdev;	state Exp;
branches;
next	1.23;

1.23
date	2006.05.31.12.04.41;	author debdev;	state Exp;
branches;
next	1.22;

1.22
date	2006.05.30.19.37.21;	author debdev;	state Exp;
branches;
next	1.21;

1.21
date	2006.05.30.14.12.54;	author debdev;	state Exp;
branches;
next	1.20;

1.20
date	2006.05.30.10.06.52;	author debdev;	state Exp;
branches;
next	1.19;

1.19
date	2006.05.30.08.16.42;	author debdev;	state Exp;
branches;
next	1.18;

1.18
date	2006.05.30.07.51.09;	author debdev;	state Exp;
branches;
next	1.17;

1.17
date	2006.05.30.06.56.38;	author debdev;	state Exp;
branches;
next	1.16;

1.16
date	2006.05.29.14.15.28;	author debdev;	state Exp;
branches;
next	1.15;

1.15
date	2006.05.28.12.41.18;	author debdev;	state Exp;
branches;
next	1.14;

1.14
date	2006.05.27.10.39.53;	author debdev;	state Exp;
branches;
next	1.13;

1.13
date	2006.05.26.16.03.47;	author debdev;	state Exp;
branches;
next	1.12;

1.12
date	2006.05.23.16.18.10;	author debdev;	state Exp;
branches;
next	1.11;

1.11
date	2006.05.23.16.10.55;	author debdev;	state Exp;
branches;
next	1.10;

1.10
date	2006.05.23.13.56.24;	author debdev;	state Exp;
branches;
next	1.9;

1.9
date	2006.05.23.13.27.39;	author debdev;	state Exp;
branches;
next	1.8;

1.8
date	2006.05.23.11.25.10;	author debdev;	state Exp;
branches;
next	1.7;

1.7
date	2006.05.22.14.30.30;	author debdev;	state Exp;
branches;
next	1.6;

1.6
date	2006.05.22.08.35.55;	author debdev;	state Exp;
branches;
next	1.5;

1.5
date	2006.05.20.15.34.38;	author debdev;	state Exp;
branches;
next	1.4;

1.4
date	2006.05.20.11.05.43;	author debdev;	state Exp;
branches;
next	1.3;

1.3
date	2006.05.19.19.36.52;	author debdev;	state Exp;
branches;
next	1.2;

1.2
date	2006.05.19.12.25.14;	author debdev;	state Exp;
branches;
next	1.1;

1.1
date	2006.05.18.19.19.25;	author debdev;	state Exp;
branches;
next	;


desc
@@


1.245
log
@when <50MB of disk,  'prelink -u'  asks if to proceed:
to avoid this, redirect stdin from /dev/null and report failures
@
text
@#!/usr/bin/python

# Copyright (C) 2006-09 Andrea Mennucci.
# License: GNU Library General Public License, version 2 or later

EMAIL="mennucc1@@debian.org"

doc={}
doc['delta']="""\
Usage: debdelta [ option...  ] fromfile tofile patchout
  Computes a delta from fromfile to tofile and writes it to patchout

Options:
--signing-key KEY
            key used to sign the delta (using GnuPG)
--no-md5    do not include MD5 info in debdelta
--needsold  create a patch that can only be used if the old .deb is available
 -M Mb      maximum memory  to use (for 'bsdiff' or 'xdelta')
--delta-algo ALGO
            use a specific backend for computing binary diffs;
"""


doc['deltas']="""\
Usage: debdeltas [ option...  ]  [deb_files and dirs, or Package files]
  Computes all missing deltas for Debian files.
  It orders by version number and produce deltas to the newest version

Options:
--signing-key KEY
            key used to sign the deltas (using GnuPG)
--dir DIR   force saving of deltas in this DIR
            (otherwise they go in the dir of the newer deb_file)

--alt DIR   for any cmdline argument, search for debs also in this dir 
            
 -n N       how many deltas to produce for each package (default 1)
--no-md5    do not include MD5 info in debdelta
--needsold  create a patch that can only be used if the old .deb is available
--delta-algo ALGO
            use a specific backend for computing binary diffs;
            possible values are: xdelta xdelta-bzip xdelta3 bsdiff
 -M Mb      maximum memory to use (for 'bsdiff' or 'xdelta')
--clean-deltas     delete deltas if newer deb is not in archive
--clean-alt        delete debs in --alt if too old (see -n )
"""

## implement : --search    search in the directory of the above debs for older versions

doc['patch']="""\
Usage: debpatch [ option...  ] patchin  fromfile  tofile 
  Applies patchin to fromfile and produces a reconstructed  version of tofile.

(When using 'debpatch' and the old .deb is not available,
  use '/' for the fromfile.)

Usage: debpatch --info  patch
  Write info on patch.

Options:
--no-md5   do not verify MD5 (if found in info in debdelta)
 -A        accept unsigned deltas
"""

doc['delta-upgrade']="""\
Usage: debdelta-upgrade [packages]
  Downloads all deltas that may be used to 'apt-get upgrade', and apply them

Options:
--dir DIR   directory where to save results
--deb-policy POLICY
            policy to decide which debs to download,
 -A         accept unsigned deltas
"""

doc['patch-url']="""\
Usage: debpatch-url [packages]
  Show URL wherefrom to downloads all deltas that may be used to 'apt-get upgrade' the given packages
"""

doc_common="""\
 -v         verbose (can be added multiple times)
--no-act    do not do that (whatever it is!)
 -d         add extra debugging checks
 -k         keep temporary files (use for debugging)
--gpg-home HOME
            specify a different home for GPG

See man page for more options and details.
"""

minigzip='/usr/lib/debdelta/minigzip'
minibzip2='/usr/lib/debdelta/minibzip2'


####################################################################

import sys , os , tempfile , string ,getopt , tarfile , shutil , time, traceback, ConfigParser, subprocess, time, tarfile, stat

from stat    import ST_SIZE, ST_MTIME, ST_MODE, S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR 
from os.path import abspath, expanduser
from copy    import copy

from types import StringType, FunctionType, TupleType, ListType, DictType

import shutil

def get_termsize():
  import termios, fcntl, struct
  s = struct.pack("HHHH", 0, 0, 0, 0)
  fd_stdout = sys.stdout.fileno()
  x = fcntl.ioctl(fd_stdout, termios.TIOCGWINSZ, s)
  return struct.unpack("HHHH", x)[:2]

try:
  (terminalrows , terminalcolumns) = get_termsize()
except:
  (terminalrows , terminalcolumns) =  (None, None) #(24, 80)

################################################# main program, read options

#target of: maximum memory that bsdiff will use
MAXMEMORY = 1024 * 1024 * 50

#this is +-10% , depending on the package size
MAX_DELTA_PERCENT = 70

#min size of .deb that debdelta will consider
#very small packages cannot be effectively delta-ed
MIN_DEB_SIZE = 10 * 1024


N_DELTAS= 1

USE_DELTA_ALGO  = 'bsdiff'

DEBUG   = 0
VERBOSE = 0
KEEP    = False
INFO    = False
NEEDSOLD= False
DIR     = None
ALT     = []
AVOID   = None
ACT     = True
DO_MD5  = True
DEB_POLICY = ['b','s','e']
DO_PROGRESS = terminalcolumns != None

#for debdeltas: test patches internally
DO_TEST = False

DO_GPG = True #this is changed a few lines below
GPG_SIGNING_KEY = None
if os.getuid() == 0:
  GPG_HOME="/etc/debdelta/gnupg"
else:
  GPG_HOME=None
GPG_MASTER_PUB_KEYRING="/usr/share/keyrings/debian-debdelta-archive-keyring.gpg"

CLEAN_DELTAS = False
CLEAN_ALT    = False

DO_PREDICTOR = False

#see README.features
DISABLEABLE_FEATURES=['lzma']
DISABLED_FEATURES=[]

RCS_VERSION="$Id: debdelta,v 1.244 2010/01/01 20:02:32 debdev Exp $"

HTTP_USER_AGENT={'User-Agent': ('Debian debdelta-upgrade' ) }

if os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
  action = None
else:
  action=(os.path.basename(sys.argv[0]))[3:]
  actions =  ('delta','patch','deltas','delta-upgrade', 'patch-url')
  
  if action not in actions:
    print 'wrong filename: should be "deb" + '+repr(actions)
    raise SystemExit(4)

  __doc__ = doc[action] + doc_common



  #GPG signatures are required for debdelta-upgrade and debpatch
  DO_GPG = action in ( "delta-upgrade", "patch")
  
  try: 
    ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' ,
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=',
                    'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug',
                    'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test") )
  except getopt.GetoptError,a:
      sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n')
      raise SystemExit(3)

  for  o , v  in  opts :
    if o == '-v' : VERBOSE += 1
    elif o == '-d' or o == '--debug' : DEBUG += 1
    elif o == '-k' : KEEP = True
    elif o == '--no-act': ACT=False
    elif o == '--no-md5': DO_MD5=False
    elif o == '--clean-deltas' : CLEAN_DELTAS = True
    elif o == '--clean-alt' : CLEAN_ALT = True
    elif o == '--needsold' :  NEEDSOLD = True
    elif o == '--delta-algo': USE_DELTA_ALGO=v
    elif o == '--max-percent': MAX_DELTA_PERCENT=int(v)
    elif o == '--deb-policy' : DEB_POLICY = [j[0] for j in v.split(',') if j]
    elif o == '-M' :
      if int(v) <= 1:
        print 'Error: "-M ',int(v),'" is too small.'
        raise SystemExit(3)
      if int(v) <= 12:
        print 'Warning: "-M ',int(v),'" is quite small.'
      MAXMEMORY = 1024 * 1024 * int(v)
    elif o == '-n' :
      N_DELTAS = int(v)
      if N_DELTAS <= 0:
        print 'Error: -n ',v,' is negative or zero.'
        raise SystemExit(3) 
    elif o == '--test' and action == 'deltas' : DO_TEST = True
    elif o == '--info' and action == 'patch' : INFO = True
    elif o == '--avoid'  :
      AVOID = v
      if not os.path.isfile(AVOID):
        print 'Error: --avoid ',AVOID,' does not exist.'
        raise SystemExit(3)
    elif o == '--dir'  :
      DIR = abspath(expanduser(v))
      if v[-2:] == '//':
        DIR += '//'
      if not os.path.isdir(DIR):
        print 'Error: --dir ',DIR,' does not exist.'
        raise SystemExit(3)
    elif o == '--alt'  :
      ALT.append(v)
      if not os.path.exists(v) :
        print 'Error: --alt ',v,' does not exist.'
        raise SystemExit(3)
    elif o ==  '--help' or o ==  '-h':
      print __doc__
      raise SystemExit(0)
    elif (o ==  '--disable-feature') and action in ("delta", "deltas"):
      DISABLED_FEATURES += v.split(',')
    elif (o ==  '--signing-key') and action in ("delta", "deltas"):
      GPG_SIGNING_KEY=v
      DO_GPG=True
    elif (o ==  '--accept-unsigned' or o == '-A') and action in ("delta-upgrade", "patch"):
      DO_GPG=False
    elif (o ==  '--gpg-home'):
      GPG_HOME=abspath(expanduser(v))
      if not os.path.isdir(GPG_HOME):
        print 'Error: --gpg-home ',GPG_HOME,' does not exist.'
        raise SystemExit(3)
    else:
      print ' option ',o,'is unknown, try --help'
      raise SystemExit(3)

for i in DISABLED_FEATURES:
  if i not in DISABLEABLE_FEATURES:
    print ' feature ',i,' cannot be disabled.'
    raise SystemExit(3)

try:
  BOGOMIPS=float(subprocess.Popen('grep bogomips /proc/cpuinfo',
                                  shell=True, stdout=subprocess.PIPE).
                 stdout.read().split(':')[-1])
except:
  if VERBOSE:
    print ' Warning, /proc not mounted, using bogus BOGOMIPS'
  BOGOMIPS=3000.0

TMPDIR = ( os.getenv('TMPDIR') or '/tmp' ).rstrip('/')

if KEEP:
  def unlink(a):
    if VERBOSE > 2: print '   -k: would unlink ',a
  def rmdir(a):
    if VERBOSE > 2: print '   -k: would rmdir ',a
  def rmtree(a):
    if VERBOSE > 2: print '   -k: would rm -r ',a
else:
  def __wrap__(a,cmd):
    c=cmd.__name__+"("+a+")"
    if a[ : len(TMPDIR)+9 ] != TMPDIR+'/debdelta' :
      raise DebDeltaError,'Internal error! refuse to  '+c
    try:
      cmd(a)
    except OSError,s:
      print ' Warning! when trying to ',repr(c),'got OSError',repr(str(s))
      raise

  def unlink(a):
    return __wrap__(a,os.unlink)
  def rmdir(a):
    return __wrap__(a,os.rmdir)
  def rmtree(a):
    return __wrap__(a,shutil.rmtree)

#################################################### various routines

def my_popen_read(cmd):
  return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout

def freespace(w):
  assert(os.path.exists(w))
  try:
    a=os.statvfs(w)
    freespace= long(a[0]) * long(a[4])
  except Exception, s:
    print 'Statvfs error:', str(s)
    freespace=None
  return freespace

dpkg_keeps_controls = (
  'conffiles','config','list','md5sums','postinst',
  'postrm','preinst','prerm','shlibs','templates')

def parse_dist(f,d):
  a=f.readline()
  p={}
  while a:
    if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File','Size','MD5s'):
      a=de_n(a)
      i=a.index(':')
      assert(a[i:i+2] == ': ')
      p[a[:i]] = a[i+2:]
    elif a == '\n':
      d[p['Package']] = p
      p={}
    a=f.readline()


def scan_control(p,params=None,prefix=None,info=None):
  if prefix == None:
    prefix = ''
  else:
    prefix += '/'
  a=p.readline()
  while a:
    a=de_n(a)
    if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File'):
      if info != None :
        info.append(prefix+a)
      if params != None:
        i=a.index(':')
        assert(a[i:i+2] == ': ')
        params[prefix+a[:i]] = a[i+2:]
    a=p.readline()

def append_info(delta,info):
  "insert into the delta (that is an AR archive) the info file, as a first element, possibly removing a previous occurrence"
  #new style : special info file
  TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
  infofile=open(TD+'/info','w')
  for i in info:
    infofile.write(i+'\n')
  infofile.close()
  if DO_GPG:
    r=_compute_hashes_(TD+"/info")
  else:
    r=None
  system(['ar','rSi','0',delta, 'info'],  TD)
  rmtree(TD)
  return r
  
def de_n(a):
  if a and a[-1] ==  '\n' :
    a = a[:-1]
  return a

def de_bar(a):
  if a and a[:2] == './' :
    a=a[2:]
  if a and a[0] == '/' :
    a=a[1:]
  return a

def list_ar(f):
  assert(os.path.exists(f))
  ar_list = []
  p=my_popen_read('ar t '+f)
  while 1:
    a=p.readline()
    if not a : break
    a=de_n(a)
    ar_list.append(a)    
  p.close()
  return ar_list

def list_tar(f):
  assert(os.path.exists(f))
  ar_list = []
  p=my_popen_read('tar t '+f)
  while 1:
    a=p.readline()
    if not a : break
    a=de_n(a)
    ar_list.append(a)    
  p.close()
  return ar_list

#####################################################################

ALLOWED = '<>()[]{}.,;:!_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'

ECHO_TEST = r"""c='\0151\0141'
E='echo -ne'
if test c`$E 'i'"$c" `o = ciiao  ; then
 :
else
 E='echo -n'
 if test c`$E 'i'"$c" `o = ciiao  ; then 
  :
 else
  #echo WARNING : BUILTIN echo DOES NOT WORK OK
  E='/bin/echo -ne'
  test c`$E 'i'"$c" `o = ciiao  
 fi
fi
"""

def prepare_for_echo__(s):
  assert ( type (s) == StringType )
  r=''
  shortquoted=False
  for a in s:
    if a in ALLOWED :
      r += a
      shortquoted = False
    elif a in '0123456789' :
      if shortquoted :
        a = "\\" + ('000' +oct(ord(a)))[-4:]
      shortquoted = False
      r += a
    else:
      a = "\\" + oct(ord(a))
      r += a
      shortquoted = len(a) < 5
  return r

def apply_prepare_for_echo(shell,repres):
    a=ECHO_TEST  + " $E '" + repres +  "' \n exit "
    p = subprocess.Popen([shell], stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
    (i, o) = (p.stdout, p.stdin)
    o.write(a)
    o.close()
    a=i.read()
    i.close()
    return a

#ack! I wanted to use 'dash' as preferred shell, but bug 379227 stopped me
SHELL = '/bin/bash'
#check my code
s='\x00'+'1ciao88\n77\r566'+'\x00'+'99\n'
r=prepare_for_echo__(s)
a=apply_prepare_for_echo(SHELL,r)
if a != s :
    print 'string='+repr(s)
    print 'repres='+repr(r)
    print 'shell='+SHELL
    print 'output='+repr(a)
    print 'Errror in prepare_for_echo.'
    raise SystemExit(4)
del r,s,a

###

def prepare_for_echo(s):
    r=prepare_for_echo__(s)
    if DEBUG > 2 :
        a=apply_prepare_for_echo(SHELL,r)
        if a != s:
            z = 'Error in prepare_for_echo()\n'
            z += 'string='+repr(s)+'\n'
            z += 'repres='+repr(r)+'\n'
            z += 'shell='+SHELL+'\n'
            z += 'output='+repr(a)+'\n'
            raise DebDeltaError(z, exitcode=4)
    return r

#####################################################################

from string import join

def version_mangle(v):
  if  ':' in v :
    return join(v.split(':'),'%3a')
  else:
    return v
  
def version_demangle(v):
  if  '%' in v :
    return join(v.split('%3a'),':')
  else:
    return v
  
def tempo():
  TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
  for i in 'OLD','NEW','PATCH' :
    os.mkdir(TD+'/'+i)
  if  VERBOSE > 2 or KEEP :  print 'Temporary in '+TD
  return TD

##########


class DebDeltaError(Exception):  #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html
  # Subclasses that define an __init__ must call Exception.__init__
  # or define self.args.  Otherwise, str() will fail.
  def __init__(self,s,retriable=False,exitcode=None):
    assert(type(s) == StringType)
    self.retriable = retriable
    if retriable:
      self.args=(s + ' (retriable) ',)
    else:
      self.args=(s + ' (non retriable) ',)
    if exitcode == None:
      if retriable:
        exitcode = 1
      else:
        exitcode = 2
    self.exitcode=exitcode

def die(s):
  #if s : sys.stderr.write(s+'\n')
  assert(type(s) == StringType)
  raise DebDeltaError,s


def system(a,TD,saveargs=None,ignore_output=False,return_output=False):
  " if return_output , it will return (stdout,stderr,exitcode) regardless"
  if type(a) != StringType :
    a=string.join(a,' ')
  if VERBOSE and TD[: (len(TMPDIR)+9) ] != TMPDIR+'/debdelta' :
    print ' Warning "system()" in ',TD,' for ',a
  (temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_out_system")
  (temp_err_fd, temp_err_name) = tempfile.mkstemp(prefix="debdelta_err_system")
  if VERBOSE > 3 : print '    system(',a,')=',
  ret = os.system("cd '" +TD +"' ; ( "+a+" ) > "+temp_name+" 2> "+temp_err_name)
  if VERBOSE > 3 : print ret
  if ignore_output==False and (os.stat(temp_name)[ST_SIZE] > 0 or os.stat(temp_err_name)[ST_SIZE] > 0 ):
    print ' command "%s" returned %d and  produced output as follows' % (a,ret)
    for i in open(temp_name):
      print 'stdout:  ',repr(i)
    for i in open(temp_err_name):
      print 'stderr:  ',repr(i)
  os.close(temp_fd)
  os.close(temp_err_fd)
  if return_output:
    return temp_name, temp_err_name, ret
  os.unlink(temp_err_name)
  os.unlink(temp_name)
  if ret == 0:
    return
  elif ret == 2:
    raise KeyboardInterrupt
  elif ret != 256 or a[:6] != 'xdelta' :
    s='Error , non zero return status '+str(ret)+' for command "'+a+'"'
    try:
      if DEBUG and saveargs:
        T=abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
        open(T+'/command','w').write(a)
        for l in saveargs:
          if l[0] != '/':
            l = TD+'/'+l
          if os.path.exists(l):
            shutil.copy2(l,T)            
            s=s+'\n saved argument '+l+' in '+T
          else:
            s=s+'\n did not find argument '+l
    except OSError,o:
      s=s+'\n    (there was an additional OSError "'+str(o)+'" when trying to save arguments)'
    die(s)

def check_deb(f):
  if not  os.path.isfile(f) :
    die('Error: '+f + ' does not exist.')
  p=open(f)
  if p.read(21) != "!<arch>\ndebian-binary" :
    die('Error: '+f+ ' does not seem to be a Debian package ')
  p.close()

def check_is_delta(f):
  if not  os.path.isfile(f) :
    die('Error: '+f + ' does not exist.')
  p=open(f)
  if p.read(8) != "!<arch>\n" :
    die('Error: '+f+ ' does not seem to be a Debian delta ')
  p.close()

def puke(s,e=''):
  " write informations on stderr, if DEBUG also traceback"
  (typ, value, trace)=sys.exc_info()
  sys.stderr.write(str(s)+' : '+str(e)+str(typ)+str(value)+'\n')
  if DEBUG and trace and traceback.print_tb(trace):
    sys.stderr.write( traceback.print_tb(trace)+'\n')

################################################################### GPG

if GPG_HOME:
  GPG_BASE_CMD_LINE=["gpg","--homedir",GPG_HOME]
else:
  GPG_BASE_CMD_LINE=["gpg","--keyring",GPG_MASTER_PUB_KEYRING]

if not VERBOSE:
  GPG_BASE_CMD_LINE+=['--quiet']

GPG_SIGN  =GPG_BASE_CMD_LINE+["--sign","--armor","--clearsign","--default-key",GPG_SIGNING_KEY]


def _compute_hashes_(na):
  "hash the file"
  #FIXME this is not present in older Python
  import hashlib
  o = open(na)
  m=hashlib.md5()
  s=hashlib.sha1()
  a=o.read(1024)
  while a:
    m.update(a)
    s.update(a)
    a=o.read(1024)
  r = ( m.hexdigest(), s.hexdigest(), os.stat(na)[ST_SIZE])
  return r

def _compute_hashes_db_(li,DIR):
  db={}
  for na in li:
    db[na] = _compute_hashes_(DIR+'/'+na)
  return db

def verify_signature(signature, DIR):
  a="-----BEGIN PGP SIGNED MESSAGE-----\n"
  if open(signature).read(len(a)) != a:
    return ('BAD_FORMAT',signature)

  role=os.path.basename(signature)
  assert  role[:4] == "_gpg"
  role=role[4:]

  (temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_gpg_verified")
  #(read_end, write_end) = os.pipe()
  p=subprocess.Popen(GPG_BASE_CMD_LINE+['--status-fd',"2",'--output',"-",signature],
                     stdout=subprocess.PIPE,stderr=temp_fd)
  r=_verify_signature_no_gpg(p.stdout, DIR, role)
  p.wait()
  
  os.close(temp_fd)
  
  if VERBOSE > 1 or p.returncode:
    for j in open(temp_name):
      print '  GPG> ',j,
  
  os.unlink(temp_name)
  
  if p.returncode:
    return ('GPG_VERIFY_FAILED',signature)
  
  return r

def _verify_signature_no_gpg(signature, DIR, role):
  #list stuff, skipping signatures
  dir_list = [a for a in os.listdir(DIR) if a[:4] != '_gpg']
  #compute signatures
  hashes = _compute_hashes_db_(dir_list, DIR)
  #scan hashes file (GPG already verified)
  if type(signature) in (str,unicode):
    f=open(signature) 
  elif hasattr(signature,'readline'):
    f=signature
  else: raise AssertionError
  a=f.readline()
  if a != "Version: 4\n":
    return ("UNSUPPORTED_VERSION",a)
  a=f.readline()
  while a:
    if a[:5] == "Role:":
      if a[5:].strip() != role :
        return ("ROLE_MISMATCH",a)
      a=f.readline()
    elif a[:6] == "Files:" :
      #parse files
      a=f.readline()
      while a and  a[0] in ( '\t' , ' ') :
        a=a.rstrip('\n')
        a=a.lstrip()
        a=a.split(' ')
        if VERBOSE > 3 : print '    checking hashes ',a
        (md5,sha1,le,na)=a
        if na not in dir_list:
          return ('ABSENT',na)
        (cmd5,csha1,cle)=hashes[na]
        if int(le) != cle:
          return ('SIZE',na)
        # check hashes
        if md5 != cmd5 :
          return ('MD5',na)
        if sha1 != csha1 :
          return ('SHA1',na)
        dir_list.remove(na)
        a=f.readline()
    elif VERBOSE > 2 :
      print '   signature header ignored: ', a
      a=f.readline()
    else:
      a=f.readline()
  #end parsing
  if dir_list:
    return ("UNCHECKED",dir_list)
  return True

def _write_signature(db,filename,role):
  "starting from a database of hashes, see _compute_hashes_, it writes a signature file"
  f=open(filename,mode='w')
  ##this is the format of dpkg-sig, but is redundant, since the "date" and "signer"
  ##are already available thru the gpg signature
  #f.write("Version: 4\nSigner: \nDate: %s\nRole: %s\nFiles: \n" % (time.ctime(),role))
  ##and actually dpkg-sig will validate also a simpler file, so, lets save a few bytes
  f.write("Version: 4\nRole: %s\nFiles:\n" % (role,))
  for a in db:
    (m,s,l) = db[a]
    f.write('\t'+m+" "+s+" "+str(l)+" "+a+"\n")
  f.close()
  
def sign_delta(delta, db, role="maker"):
  TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
  try:
    _write_signature(db,TD+'/_temp',role)
    p=subprocess.Popen(GPG_SIGN+['--output',TD+'/_gpg'+role,TD+'/_temp'])
    p.wait()
    if p.returncode==0:
      r=os.system("ar qS "+delta+" "+TD+"/_gpg"+role)
  except:
    rmtree(TD)
    raise
  rmtree(TD)
  if p.returncode:
    raise DebDeltaError('GnuPG fails to sign')
  if r:
    raise DebDeltaError('ar fails to add the signature')




#################################################################### apply patch

########### info auxiliary routines

def _info_patch_unzip_(TD):
  "unzip info and patch.sh"
  if os.path.exists(TD+'PATCH/info.gz'):
    system('gunzip PATCH/info.gz',TD)
  if os.path.exists(TD+'PATCH/patch.sh.gz'):
    system('gunzip PATCH/patch.sh.gz',TD)
  elif os.path.exists(TD+'PATCH/patch.sh.bz2'):
    system('bunzip2 PATCH/patch.sh.bz2',TD)  
  elif os.path.exists(TD+'PATCH/patch.sh.lzma'):
    if not os.path.exists('/usr/bin/unlzma'):
      raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True)
    system('unlzma PATCH/patch.sh.lzma',TD)

def get_info_slow(delta,T=None):
  if T:
    TD=T
  else:
    TD=tempo()
  if TD[-1] != '/':
    TD = TD + '/'
  delta=abspath(expanduser(delta))
  system('ar x  '+delta+' info info.gz patch.sh patch.sh.gz patch.sh.bz2',
         TD+'/PATCH', ignore_output=True)
  _info_patch_unzip_(TD)
  info = _scan_delta_info_(TD)
  if T == None:
    rmtree(TD)
  return info

def get_info_fast(delta):
  f=open(delta)
  s=f.readline()
  if  "!<arch>\n" != s :
    raise DebDeltaError('This is not a debdelta file: '+delta)
  s = f.read(60)
  if len(s) != 60 :
    print '(Warning, cannot get info from  truncated: '+delta+' )'
    return None
  if s[:4] != 'info':
    #old style debdelta, with info in patch.sh
    if VERBOSE > 1 : print '  (Warning, cannot get info from old style: '+delta+' )'
    return None
  ##parse ar segment
  ## see /usr/include/ar.h
  if s[-2:] != '`\n' :
    print '(Warning, cannot get info from  '+delta+' , format not known)'
    return None
  l=int(s[ -12:-2 ])
  s=f.read(l)
  if len(s) != l :
    print '(Warning, cannot get info from truncated: '+delta+' )'
    return None
  info= s.split('\n')
  f.close()
  return info

def get_info(delta,TD=None):
  info=get_info_fast(delta)
  if info == None:
    info=get_info_slow(delta,TD)
  return info

def _scan_delta_info_(TD):
    info=[]
    if os.path.isfile(TD+'PATCH/info'):
      #new style debdelta, with info file
      p=open(TD+'PATCH/info')
      info=p.read().split('\n')
      p.close()
      if info[-1] == '': info.pop()
    else:
      #old style debdelta, with info in patch.sh
      p=open(TD+'PATCH/patch.sh')
      s=p.readline()
      s=p.readline()
      while s:
        if s[0] == '#' :
          s=de_n(s)
          info.append(s[1:])
        s=p.readline()
      p.close()
    return info

def info_2_db(info):
  params={}
  for s in info:
    if ':' in s:
      i=s.index(':')  
      params[s[:i]] = s[i+2:]
    elif s:
      params[s] = True
  return params

########### other auxiliary routines

def patch_check_tmp_space(params,olddeb):
  if type(params) != DictType:
    params=info_2_db(params)
  if 'NEW/Installed-Size' not in params or 'OLD/Installed-Size' not in params:
    print '(Warning... Installed size unknown...)'
    return True
  free=freespace(TMPDIR)
  if free == None : return True
  free = free / 1024
  if olddeb == '/':
    instsize=int(params['NEW/Installed-Size'])
    #the last action of the script is to gzip the data.tar, so
    if 'NEW/Size' in params :
      instsize += int(params['NEW/Size']) / 1024
    else:
      instsize = instsize * 1.8
  else:
    instsize=int(params['NEW/Installed-Size'])+int(params['OLD/Installed-Size'])
  instsize +=  2**13
  if free <  instsize :
    return 'not enough disk space (%dkB) in %s for applying delta (needs %dkB).' % \
        ( int(free) , TMPDIR, instsize )
  else:
    return True


def scan_diversions():
  f=open('/var/lib/dpkg/diversions')
  d={}

  a=1
  while 1:
    a=f.readline()
    if not a: break
    a=de_n(a)
    b=de_n(f.readline())
    p=de_n(f.readline())
    d[a]=(b,p)
  f.close()
  return d

###################################################### debforensic extract

#in base-passwd 3.5.11
#/usr/share/base-passwd/passwd.master
base_passwd="""root::0:0:root:/root:/bin/bash
daemon:*:1:1:daemon:/usr/sbin:/bin/sh
bin:*:2:2:bin:/bin:/bin/sh
sys:*:3:3:sys:/dev:/bin/sh
sync:*:4:65534:sync:/bin:/bin/sync
games:*:5:60:games:/usr/games:/bin/sh
man:*:6:12:man:/var/cache/man:/bin/sh
lp:*:7:7:lp:/var/spool/lpd:/bin/sh
mail:*:8:8:mail:/var/mail:/bin/sh
news:*:9:9:news:/var/spool/news:/bin/sh
uucp:*:10:10:uucp:/var/spool/uucp:/bin/sh
proxy:*:13:13:proxy:/bin:/bin/sh
www-data:*:33:33:www-data:/var/www:/bin/sh
backup:*:34:34:backup:/var/backups:/bin/sh
list:*:38:38:Mailing List Manager:/var/list:/bin/sh
irc:*:39:39:ircd:/var/run/ircd:/bin/sh
gnats:*:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/bin/sh
nobody:*:65534:65534:nobody:/nonexistent:/bin/sh"""
base_passwd_db={}
base_passwd_anti_db={}
for a in base_passwd.split('\n'):
    a=a.split(':')
    base_passwd_db[a[0]]=int(a[2])
    base_passwd_anti_db[int(a[2])]=a[0]

base_group="""root:*:0:
daemon:*:1:
bin:*:2:
sys:*:3:
adm:*:4:
tty:*:5:
disk:*:6:
lp:*:7:
mail:*:8:
news:*:9:
uucp:*:10:
man:*:12:
proxy:*:13:
kmem:*:15:
dialout:*:20:
fax:*:21:
voice:*:22:
cdrom:*:24:
floppy:*:25:
tape:*:26:
sudo:*:27:
audio:*:29:
dip:*:30:
www-data:*:33:
backup:*:34:
operator:*:37:
list:*:38:
irc:*:39:
src:*:40:
gnats:*:41:
shadow:*:42:
utmp:*:43:
video:*:44:
sasl:*:45:
plugdev:*:46:
staff:*:50:
games:*:60:
users:*:100:
nogroup:*:65534:"""

base_group_db={}
base_group_anti_db={}
for a in base_group.split('\n'):
    a=a.split(':')
    base_group_db[a[0]]=int(a[2])
    base_group_anti_db[int(a[2])]=a[0]

# all code following return name,mode,tartype,uid,gid,uname,gname

#adapted from tarfile.py, a Python module
def stat_to_tar(name):
    "returns name,mode,tartype,uid,gid,uname,gname,data"
    statres = os.lstat(name)
    stmd = statres.st_mode
    data = None
    if stat.S_ISREG(stmd):
        tartype = tarfile.REGTYPE
        # here ideally we should SHA1 the file ; 
        # but this is done elsewhere for performance, 
        # and to have multi_hash in the future
    elif stat.S_ISDIR(stmd):
        tartype = tarfile.DIRTYPE
    elif stat.S_ISFIFO(stmd):
        tartype = tarfile.FIFOTYPE
    elif stat.S_ISLNK(stmd):
        tartype = tarfile.SYMTYPE
        data = os.readlink(name)
    elif stat.S_ISCHR(stmd):
        tartype = tarfile.CHRTYPE
    elif stat.S_ISBLK(stmd):
        tartype = tarfile.BLKTYPE
    elif stat.S_ISSOCK(stmd):
        tartype = 'SOCKET'  #SOCKETs are not supported in tar files
    else: raise TypeError
    
    if tartype in (tarfile.CHRTYPE, tarfile.BLKTYPE):
        data = str(os.major(statres.st_rdev))+' '+str( os.minor(statres.st_rdev))

    uid,gid = statres.st_uid, statres.st_gid 
    
    if uid in base_passwd_anti_db :
        uname = base_passwd_anti_db[uid]
    else:
        import pwd
        try:
            uname = pwd.getpwuid(uid)[0]
        except KeyError:
            uname = None

    if gid in base_group_anti_db :
        gname = base_group_anti_db[gid]
    else:
        import grp
        try:
            gname = grp.getgrgid(gid)[0]
        except KeyError:
            gname = None

    #07777 is used in tarfile.TarInfo.tobuf
    return  name.lstrip('/'), stmd & 07777, tartype, uid, gid, uname, gname, data


def tarinfo_to_ls(tartype,tarmode):
    "returns a string -rwxrwxrwx such as what ls -l prints "
    if ord(tartype) == 0 :
        a='_'
    else:
        if tartype >= '0' and tartype <= '6' :
            a="-hlcbdp"[ord(tartype) - ord('0')] 
        else:
            a='?'
    return a+tarfile.filemode(tarmode)[1:]


def sha1_hash_file(f):
    #FIXME this is not present in older Python
    import hashlib
    s=hashlib.sha1()
    if type(f) == StringType:
      f=open(f)
    a=f.read(1024)
    while a:
      s.update(a)
      a=f.read(1024)
    f.close()
    return s.digest()

def hash_to_hex(s):
    a=''
    for i in s:
        a=a+ ( '%02x' % ord(i)  )
    return a

def forensics_rfc(o,db,controlfiles,files,diverted,diversions,conffiles=[]):
  o.write('Package: '+db['OLD/Package']+'\n')
  o.write('Version: '+db['OLD/Version']+'\n')
  o.write('Architecture: '+db['OLD/Architecture']+'\n')
  if diverted:
    o.write("Diversions:\n")
    for a in diverted:
      b,p = diversions[a]
      o.write(" From: "+a+'\n')
      o.write(" To: "+b+'\n')
      o.write(" By: "+p+'\n')
  if conffiles:
    o.write("Conffiles:\n")
    for a in conffiles:
      o.write(' '+a+'\n')
  for L,N in ((controlfiles,"Control"),(files,"Files")):
    o.write(N+":\n")
    for a,b in L:
      if not os.path.exists(b):
        o.write(' NONEXISTENT\n '+b+'\n \n')
        continue
      name,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(b)
      if tartype == tarfile.REGTYPE:
        data=hash_to_hex(sha1_hash_file(b))
      if uname == None: uname=str(uid)
      if gname == None: gname=str(gid)
      o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname)
      if N == "Files" and tartype == tarfile.REGTYPE and a in conffiles:
        o.write(" [conffile]\n")
      else:
        o.write("\n")
      o.write(" "+a+"\n")
      if data!=None:
        o.write(" "+data+"\n")
      else:
        o.write(" \n")

def elf_info(f):
  "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)"
  import struct
  elfheader=open(f).read(32)
  if len(elfheader) == 32:
    #parse as specified in /usr/include/elf.h from libelf-dev
    EI_CLASS={1:'ELFCLASS32',
              2:'ELFCLASS64'}
    EI_DATA={1:'ELFDATA2LSB', #  2's complement, little endian 
             2:'ELFDATA2MSB'} #  2's complement, big endian
    EI_OSABI={0:'ELFOSABI_SYSV',# UNIX System V ABI
              1:'ELFOSABI_HPUX',
              2:'ELFOSABI_NETBSD',
              3:'ELFOSABI_LINUX',
              #fixme insert other values
              9:'ELFOSABI_FREEBSD',
              12:'ELFOSABI_OPENBSD',
              97:'ELFOSABI_ARM'}
    #fixme what is ET_LOOS , ET_HIOS  , ET_LOPROC, ET_HIPROC ??
    ET_TYPE={1:'ET_REL', #Relocatable file
             2:'ET_EXEC', #Executable file
             3:'ET_DYN',  #Shared object file
             4:'ET_CORE'} #Core file
    
    ei_magic, ei_class, ei_data, ei_version, ei_osabi, ei_abiversion = \
        struct.unpack_from('4sBBBBB',elfheader)
    e_type, e_machine, e_version = struct.unpack_from('HHI',elfheader,16)
    #FIXME I think I am getting ei_osabi wrong.. it is always 0
    is_elf = '\x7fELF' == ei_magic 
    #and ei_class in (1,2) and \
    #    ei_version == 1 and \
    #    ei_data in (1,2) and e_type>0 and e_machine>0 and e_version>0
    return is_elf, EI_CLASS.get(ei_class), \
        EI_DATA.get(ei_data), EI_OSABI.get(ei_osabi), ET_TYPE.get(e_type)
  #, e_machine, e_version
  else:
    return False, 0, 0, 0, 0

def parse_prelink_conf():
    " fixme , currently unused and incomplete " 
    prelinked_dirs=[]
    prelinked_blacklist=[]
    prelinked_blacklist_glob=[]
    for a in open('/etc/prelink.conf'):
      if a[0] == '#':
        continue
      a=a.strip()
      b=a.split()
      if len(b) != 2:
        print '  (sorry this line of /etc/prelink.conf cannot be parsed currently: "'+a+'")'
        continue
      if '-b' == b[0]:
        if '/' in b[1]:
          prelinked_blacklist.append(b[1])
        else:
          prelinked_blacklist_glob.append(b[1])
      elif '-l' == b[0]:
        prelinked_dirs.append(b[1])
  

############ do_patch

def do_patch(delta,olddeb,newdeb, info=None, diversions=None, do_gpg=DO_GPG):
  runtime={}
  try:
    T=tempo()
    r=do_patch_(delta,olddeb,newdeb, T, runtime,
                info=info, diversions=diversions, do_gpg=do_gpg)
  except:
    rmtree(T)
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    raise
  rmtree(T)
  return r

def do_patch_(delta, olddeb, newdeb, TD, runtime,
              info=None, diversions=None, do_gpg=DO_GPG, do_progress=DO_PROGRESS):
  if TD[-1] != '/':
    TD = TD + '/'
  
  HAVE_PRELINK=os.path.exists('/usr/sbin/prelink')
  HAVE_LOCALEPURGE=os.path.exists('/etc/locale.nopurge') or os.path.exists('/usr/sbin/localepurge')
  
  delta=abspath(delta)
  newdebshortname='-'
  if newdeb:
    newdebshortname=newdeb
    newdeb=abspath(newdeb)
  if olddeb != '/':
    olddeb=abspath(olddeb)
  elif diversions == None:
    diversions=scan_diversions()
  
  
  start_sec = time.time()
  runtime['patchprogress']=0
  
  check_is_delta(delta)

  if olddeb != '/':
      check_deb(olddeb)
  
  temp_name, temp_err_name, ret=system('ar xvo '+delta,  TD+'/PATCH', return_output=True, ignore_output=True)
  if ret :
    raise DebDeltaError('Cannot extract from '+delta)
  ar_list_delta=[a[4:] for a in open(temp_name).read().split('\n') if a]
  os.unlink(temp_name)
  os.unlink(temp_err_name)

  runtime['patchprogress']=1

  is_signed=False
  for a in ar_list_delta:
    if a[:4] == '_gpg':
      r = verify_signature(TD+'/PATCH/'+a,TD+'/PATCH')
      if True != r:
        die(delta+": the signature file "+a+" fails as follows: "+repr(r))
      is_signed=True
      if VERBOSE : print ' The signature by "'+a[4:]+'" is correctly verified for ',delta
  if not is_signed:
    if do_gpg:
      die("Delta is not signed: "+delta)
    elif do_gpg != None:
      print "WARNING, delta is not signed: "+delta

  runtime['patchprogress']=2

  _info_patch_unzip_(TD)

  if not os.path.isfile(TD+'PATCH/patch.sh'):
    die('Error. File '+delta+' is not a debdelta file.')

  os.symlink(minigzip,TD+'minigzip')
  os.symlink(minibzip2,TD+'minibzip2')
  
  #lets scan parameters, to see what it does and what it requires
  if info == None :
      info=_scan_delta_info_(TD)
  params=info_2_db(info)
  
  runtime['patchprogress']=3

  #this is not needed in preparing the patch, but may help in forensic
  conf_files=[]
  a='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles'
  if DEBUG and os.path.isfile(a):
    #note that filenames have leading /
    conf_files=[p for p in open(a).read().split('\n') if p]
  del a
  
  ###
  s=patch_check_tmp_space(params,olddeb)
  if s != True:
    raise DebDeltaError('Sorry, '+s, True )

  if olddeb != '/':
      os.symlink(olddeb,TD+'/OLD.file')
      #unpack the old control structure, if available
      os.mkdir(TD+'/OLD/CONTROL')
      #unpack control.tar.gz
      system('ar p '+TD+'OLD.file control.tar.gz | tar -x -z -p -f - -C '+TD+'OLD/CONTROL',TD)
  #then we check for the conformance
  if olddeb != '/' and 'OLD/Size' in params:
    olddebsize = os.stat(olddeb)[ST_SIZE]
    if olddebsize != int(params['OLD/Size']):
      raise DebDeltaError('Old deb size is '+str(olddebsize)+' instead of '+params['OLD/Size'])
  
  runtime['patchprogress']=4

  if DEBUG > 1 :
      #this is currently disabled, since  'dpkg -s' is vey slow (~ 1.6 sec)
      dpkg_params={}
      b=params['OLD/Package']
      if olddeb == '/' :
        p=my_popen_read('env -i dpkg -s '+b)
      else:        
        p=open(TD+'OLD/CONTROL/control')
      scan_control(p,params=dpkg_params,prefix='OLD')
      p.close()
      if  olddeb == '/' :
        if 'OLD/Status' not in dpkg_params:
          die('Error: package %s is not known to dpkg.' % b)
        if  dpkg_params['OLD/Status'] != 'install ok installed' :
          die('Error: package %s is not installed, status is %s.'
            % ( b , dpkg_params['OLD/Status'] ) )
      for a in  params:
        if a[:3] == 'OLD' and a != 'OLD/Installed-Size' and a != 'OLD/Size':
          if a not in dpkg_params:
            die('Error parsing old control file , parameter %s not found' % a)
          elif  params[a] != dpkg_params[a] :
            die( 'Error : in debdelta , '+a+' = ' +params[a] +\
                 '\nin old/installed deb, '+a+' = ' +dpkg_params[a])

  runtime['patchprogress']=5

  ### some auxiliary routines, separated to make code more readable

  def dpkg_L_faster(pa,diversions):
    "Scan dpkg -L . 'diversions' must be prepared by scan_diversions() . Returns list of pairs of files ,and list of diverted files. "
    s=[]
    diverted=[]
    f=open('/var/lib/dpkg/info/'+pa+'.list')
    while 1:
      a=f.readline()
      if not a: break
      a=de_n(a)
      if a in diversions:
        b,p= diversions[a]
        if p != pa:
          s.append((a,b))
          diverted.append(a)
        else:
          s.append((a,a))
      else: s.append((a,a))
    f.close()
    return s,diverted

  def dpkg_L(pa):
    "Scan dpkg -L . Currently unused, see previous function."
    sys.stderr.write('INTERNAL WARNING: USING OBSOLETE dpkg_L\n')
    s=[]
    diverted=[]
    p=my_popen_read('env -i dpkg -L '+pa)
    a=p.readline()
    while a:
      a=de_n(a)
      #support diversions
      if a[:26] == 'package diverts others to:':
        continue
      if s and a[:11] == 'diverted by' or  a[:20] == 'locally diverted to:':
        orig,divert=s.pop()
        i = a.index(':')
        divert = a[i+2:]
        s.append( (orig,divert) )
        diverted.append(orig)
      else:
        s.append( (a,a) )
      a=p.readline()
    p.close()
    return s, diverted

  localepurged=[]
  def _symlink_data_tree(pa,TD,diversions,runtime):
    if diversions:
      s,diverted=dpkg_L_faster(pa,diversions)
    else:
      s,diverted=dpkg_L(pa)
    progressline=0
    progresslen=float(len(s))
    for orig,divert in s:
      progressline+=1
      progress=6.0 + 6.0 * float(progressline) / progresslen
      runtime['patchprogress']=progress
      if do_progress:
        sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname))
      if os.path.isfile(divert) and not os.path.islink(divert) :
        a=TD+'OLD/DATA'+orig
        d=os.path.dirname(a)
        if not os.path.exists(d):
          os.makedirs(d)
        #the following code idea was provided by roman@@khimov.ru
        unprelink=False
        if HAVE_PRELINK :
          is_elf, ei_class, ei_data, ei_osabi, e_type = elf_info(divert)
          #according to prelink-0.0.20090925/src/main.c
          unprelink = is_elf and e_type in ('ET_DYN','ET_EXEC')
        if unprelink:
          shutil.copyfile(divert, a)
          if VERBOSE > 3 :
            print '    copying/unprelinking ',divert,' to ',a
          if DEBUG or VERBOSE:
            ret=os.system("/usr/sbin/prelink -u '"+a+"' < /dev/null")
          else:
            ret=os.system("/usr/sbin/prelink -u '"+a+"' < /dev/null >/dev/null 2>&1")
          if ret==2:
            raise KeyboardInterrupt
          elif ret:
            sys.stderr.write('!!Prelink failed, rerun with -d\n')
        else:
          if VERBOSE > 3 : print '    symlinking ',divert,' to ',a
          os.symlink(divert, a)
      elif not os.path.exists(divert):
        if VERBOSE : print ' Disappeared file? ',divert
        for z in ('locale','man','gnome/help','omf','doc/kde/HTML'):
          w='/usr/share/'+z
          if orig[:len(w)] == w:
            localepurged.append(orig)
      elif VERBOSE > 3 : print '    not symlinking ',divert,' to ',orig
    return s,diverted


  def chmod_add(n,m):
    "same as 'chmod ...+...  n '"
    om=S_IMODE(os.stat(n)[ST_MODE])
    nm=om | m
    if nm != om :
      if VERBOSE > 2 : print '   Performing chmod ',n,oct(om),oct(nm)
      os.chmod(n,nm)
  
  def _fix_data_tree_(TD):
    for (dirpath, dirnames, filenames) in os.walk(TD+'OLD/DATA'):
      chmod_add(dirpath,  S_IRUSR | S_IWUSR| S_IXUSR  )
      for i in filenames:
        i=os.path.join(dirpath,i)
        if os.path.isfile(i):
          chmod_add(i,  S_IRUSR |  S_IWUSR )
      for i in dirnames:
        i=os.path.join(dirpath,i)
        chmod_add(i,  S_IRUSR | S_IWUSR| S_IXUSR  )

  control_file_pairs=[]
  linked_file_pairs,diverted=[],[]
  
  ###see into parameters: the patch may need extra info and data

  runtime['patchprogress']=6

  for a in params:
    if 'needs-old' == a:
      if olddeb == '/':
        die('This patch needs the old version Debian package')
    elif 'old-data-tree' == a :
      os.mkdir(TD+'/OLD/DATA')
      if olddeb == '/':
        linked_file_pairs,diverted=_symlink_data_tree(params['OLD/Package'],TD,diversions,runtime)
      else:
        ar_list_old= list_ar(TD+'OLD.file')
        if 'data.tar.bz2' in ar_list_old:
          system('ar p '+TD+'OLD.file data.tar.bz2 | tar -x --bzip2 -p -f - -C '+TD+'OLD/DATA', TD)
        elif 'data.tar.gz' in ar_list_old:
          system('ar p '+TD+'OLD.file data.tar.gz | tar -x -z -p -f - -C '+TD+'OLD/DATA', TD)
        elif 'data.tar.lzma' in ar_list_old:
          if not os.path.exists('/usr/bin/lzma'):
            raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True)
          system('ar p '+TD+'OLD.file data.tar.lzma | unlzma -c | tar -x -p -f - -C '+TD+'OLD/DATA', TD)
        else: assert(0)
        _fix_data_tree_(TD)
    elif 'old-control-tree' == a:
        if olddeb == '/':
          if not os.path.isdir(TD+'OLD/CONTROL'):
            os.mkdir(TD+'OLD/CONTROL')
          p=params['OLD/Package']
          for  b in dpkg_keeps_controls :
            a='/var/lib/dpkg/info/' + p +'.'+b
            if os.path.exists(a):
              os.symlink(a,TD+'OLD/CONTROL/'+b)
              control_file_pairs.append((b,a))
        #else... we always unpack the control of a .deb
    elif 'needs-xdelta3' == a:
      if not os.path.exists('/usr/bin/xdelta3'):
        raise DebDeltaError('This patch needs xdelta3. Please install the Debian package "xdelta3".',retriable=True)
    elif 'needs-xdelta' == a:
      if not os.path.exists('/usr/bin/xdelta'):
        raise DebDeltaError('This patch needs xdelta. Please install the Debian package "xdelta".',retriable=True)
    elif 'needs-bsdiff' == a:
      if not os.path.exists('/usr/bin/bsdiff'):
        raise DebDeltaError('This patch needs bsdiff. Please install the Debian package "bsdiff".',retriable=True)
    elif 'needs-lzma' == a:
      if not os.path.exists('/usr/bin/lzma'):
        raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True)
    elif 'needs-minibzip2' == a:
      pass #its your lucky day
    elif a[:6] == 'needs-':
      raise DebDeltaError('patch says "'+a+"' and this is unsupported. Get a newer debdelta.",retriable=True)
    elif params[a] == True:
        print  'WARNING patch says "'+a+'" and this is unsupported. Get a newer debdelta.'

  if localepurged and HAVE_LOCALEPURGE and not DEBUG:
    #actually we cannot be 100% sure that the delta really needs those files, but it is quite plausible
    raise DebDeltaError('localepurge removed '+str(len(localepurged))+' files.')

  runtime['patchprogress']=12

  a=''
  if DEBUG: a='-v'
  script_time = - time.time()

  temp_err_name_fd, temp_err_name = tempfile.mkstemp(prefix='debdeltaE')
  temp_name_fd, temp_name = tempfile.mkstemp(prefix='debdeltaO')
  F=subprocess.Popen([SHELL,'-v','-e','PATCH/patch.sh'], cwd=TD,
                     stderr=subprocess.PIPE, stdout=temp_name_fd)
  progresschar=0.0
  progresslen=float(os.path.getsize(os.path.join(TD,'PATCH/patch.sh')))
  for j in F.stderr:
    os.write(temp_err_name_fd, j)
    progresschar+=len(j)
    progress=(int(12.0 + 84.0 * progresschar / progresslen))
    runtime['patchprogress']=progress
    if do_progress:
      sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname))
  F.wait()
  if do_progress and terminalcolumns: #clean up
    sys.stderr.write(' ' * (terminalcolumns-2) +'\r')
  ret=F.returncode
  os.close(temp_err_name_fd)
  os.close(temp_name_fd)

  script_time += time.time()
  runtime['patchprogress']=97

  #helper for debugging
  def tempos():
    if os.path.getsize(temp_name):
      sys.stderr.write('!! '+temp_name+'\n')
    if os.path.getsize(temp_err_name):
      sys.stderr.write('!! '+temp_err_name+'\n')

  if DEBUG == 0:
    def fore():
      sys.stderr.write('(Faulty delta. Please consider retrying with the option "-d" ).\n')
  elif olddeb != '/':
    def fore():
      sys.stderr.write('!!Faulty delta. Please send by email to '+EMAIL+' the files:\n!! '+
                       delta+'\n!! '+ olddeb+'\n')
      tempos()
  else:
    def fore():
      temp_fore_name=''
      try:
        (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_")
        temp_file=os.fdopen(temp_fd,'w')
        temp_file.write('Delta: '+delta+'\n')
        temp_file.write('DeltaSHA1: '+hash_to_hex(sha1_hash_file(delta))+'\n')
        temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n')
        if ret:
          temp_file.write('PatchExitCode: '+str(ret)+'\n')
        forensics_rfc(temp_file,params,control_file_pairs,
                      linked_file_pairs,diverted,diversions,conf_files)
        temp_file.close()
      except OSError: #Exception,s:
        die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n')
      sys.stderr.write('!!Faulty delta. Please send by email to '+EMAIL+' the files:\n!! '+
                       temp_fore_name+'\n')
      tempos()

  ##then , really execute the patch

  if ret:
    if ret == 2:
      raise KeyboardInterrupt
    elif localepurged:
      raise DebDeltaError('"debdelta" is incompatible with "localepurge".')
    else:
      fore()
      raise DebDeltaError('error in patch.sh.')

  #then we check for the conformance
  if  'NEW/Size' in params:
    newdebsize = os.stat(TD+'NEW.file')[ST_SIZE]
    if newdebsize != int(params['NEW/Size']):
      fore()
      raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'])

  if DO_MD5:
    if 'NEW/MD5sum' in params:
      if VERBOSE > 1 : print '  verifying MD5  for ',os.path.basename(newdeb or delta)
      try:
        system('echo "'+params['NEW/MD5sum']+'  NEW.file" | md5sum -c', TD,
               ignore_output=True)
      except:
        fore()
        raise
    else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb or delta)

  os.unlink(temp_name)
  os.unlink(temp_err_name)

  runtime['patchprogress']=99

  if newdeb:
      shutil.move(TD+'NEW.file',newdeb)

  end_sec = time.time()
  elaps=(end_sec - start_sec)

  if VERBOSE :
      if newdeb:
        debsize = os.stat(newdeb)[ST_SIZE]
      else:
        debsize = os.stat(olddeb)[ST_SIZE]
      a=''
      if newdeb != None:
        a='result: '+os.path.basename(newdeb)
      print ' Patching done, time: %.2fsec, speed: %dkB/sec %s (script time %.2fsec ) ' % \
            (elaps,(debsize / 1024 /  (elaps+.001)),a , script_time)
  return (newdeb,elaps)

##################################################### compute delta
def do_delta(olddeb,newdeb,delta):
  try:
    T=tempo()
    r=do_delta_(olddeb,newdeb,delta,TD=T)
    (delta, percent, elaps, info, gpg_hashes) = r
    info_hashes=append_info(delta,info)
    if DO_GPG:
      gpg_hashes['info']=info_hashes
      sign_delta(delta,gpg_hashes)
  except:
    if delta and os.path.exists(delta):
      os.unlink(delta)
    rmtree(T)
    raise
  else:
    rmtree(T)
  return r

def do_delta_(olddeb,newdeb,delta,TD):
  if TD[-1] != '/':
    TD = TD + '/'

  import fnmatch  
  
  start_sec = time.time()

  #I do not like global variables but I do not know of another solution
  global bsdiff_time, bsdiff_datasize
  bsdiff_time = 0
  bsdiff_datasize = 0
  
  olddeb=abspath(olddeb)
  check_deb(olddeb)
  os.symlink(olddeb,TD+'/OLD.file')
  olddebsize = os.stat(olddeb)[ST_SIZE]
  
  newdeb=abspath(newdeb)
  check_deb(newdeb)
  os.symlink(newdeb,TD+'/NEW.file')
  newdebsize = os.stat(newdeb)[ST_SIZE]
  
  free=freespace(TD)
  if free and free < newdebsize :
    raise DebDeltaError('Error: not enough disk space in '+TD, True)

  delta=abspath(delta)
  if  os.path.exists(delta) :
    os.rename(delta,delta+'~')
  
  #generater for numbered files
  def a_numb_file_gen():    
    deltacount = 0
    while 1:
      yield str(deltacount)
      deltacount+=1      
  a_numb_file=a_numb_file_gen()
  
  #start writing script 
  script=open(TD+'PATCH/patch.sh','w')
  script.write('#!/bin/bash -e\n')
    
  ##### unpack control.tar.gz, scan control, write  parameters
  info=[]
  def info_append(s):
    "smart appending that avoids duplicate entries"
    if s not in info:
      info.append(s)
  
  for o in 'OLD', 'NEW' :
      os.mkdir(TD+o+'/CONTROL')
      #unpack control.tar.gz
      system('ar p '+TD+o+'.file control.tar.gz | tar -x -z -f - -C '+TD+o+'/CONTROL',TD)
      ## scan control
      p=open(TD+'/'+o+'/CONTROL/control')
      s=[]
      scan_control(p,params=None,prefix=o,info=s)
      p.close()
      if  VERBOSE  : print ' '+o+': '+join([o[4:] for o in  s],' ')
      info = info + s
      del s,p
  info.append('OLD/Size: '+str(olddebsize))
  info.append('NEW/Size: '+str(newdebsize))
  params=info_2_db(info)
  
  #scan debdelta.conf to find any special requirement
  debdelta_conf=ConfigParser.SafeConfigParser()
  debdelta_conf.read(['/etc/debdelta/debdelta.conf', expanduser('~/.debdelta/debdelta.conf')  ])

  debdelta_conf_skip=[]
  for s in debdelta_conf.sections():
    if fnmatch.fnmatch(params['OLD/Package'],s):
      opt=debdelta_conf.options(s)
      if 'skip' in opt:
        debdelta_conf_skip += debdelta_conf.get(s,'skip').split(';') 
      break

  if VERBOSE > 1 : print '  debdelta.conf says we will skip: ', repr(debdelta_conf_skip)  

  gpg_hashes = {}
  
  if DO_MD5 :
    # compute a MD5 of NEW deb
    p=my_popen_read('md5sum '+TD+'NEW.file')
    a=p.readline()
    p.read()
    p.close
    newdeb_md5sum=a[:32]
    info.append('NEW/MD5sum: '+ newdeb_md5sum[:32])
  else:
    newdeb_md5sum=None

  if NEEDSOLD :
    #this delta needs the old deb 
    info.append('needs-old')
  else:
    info.append('old-data-tree')
    info.append('old-control-tree')

  a=USE_DELTA_ALGO
  if a == 'xdelta-bzip':
    a='xdelta'
  if not os.path.exists('/usr/bin/'+a):
    raise DebDeltaError('please install the package "'+a+'".', retriable=True)
  info.append('needs-'+a)
  del a

  #### check for disk space
  if 'NEW/Installed-Size' in params and 'OLD/Installed-Size' in params:
    free=freespace(TD)  
    instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size'])
    if free and free < ( instsize * 1024 + + 2**23 + MAXMEMORY / 6 ) :
      raise DebDeltaError(' Not enough disk space (%dkB) for creating delta (needs %dkB).' % \
          ( int(free/1024) , instsize ) , True )

    
  ############# check for conffiles 
  a=TD+'/OLD/CONTROL/conffiles'
  if os.path.exists(a):
    p=open(a)
    #files do not have leading /
    old_conffiles=[ de_bar(a) for a in p.read().split('\n') if a]
    p.close()
  else:
    old_conffiles=[]

##   a=TD+'/OLD/CONTROL/list'
##   if os.path.exists(a):
##     p=open(a)
##     for a in p:
##       a=de_bar(de_n(a))
##       for j in debdelta_conf_skip:
##         if fnmatch(a,j):
##           old_conffiles.append(a) #OK, this abuses the name of the var a bit
##           print ' REPR skip ',repr(a)
##   else:
##     print '  The old debian package ',olddeb,' does not contain a file list?!?' 

  def shell_not_allowed(name):
    "Strings that I do not trust to inject into the shell script; maybe I am a tad too paranoid..."
    #FIXME should use it , by properly quoting for the shell script
    return '"' in name or "'" in name or '\\' in name or '`' in name 

  # uses MD5 to detect identical files (even when renamed)
  def scan_md5(n):
    md5={}
    f=open(n)
    a=de_n(f.readline())
    while a:
      m , n = a[:32] ,  de_bar( a[34:] )
      md5[n]=m
      a=de_n(f.readline())
    f.close()
    return md5


  new_md5=None
  if os.path.exists(TD+'/NEW/CONTROL/md5sums'):
    new_md5=scan_md5(TD+'/NEW/CONTROL/md5sums')
    
  old_md5=None
  if os.path.exists(TD+'/OLD/CONTROL/md5sums') :
    old_md5=scan_md5(TD+'/OLD/CONTROL/md5sums')

  ############### some routines  to prepare delta of two files

  def script_md5_check_file(n,md5=None):
    if md5==None:
      assert(os.path.isfile(TD+n))
      pm=my_popen_read('md5sum '+TD+n)
      a=pm.readline()
      pm.read()
      md5=a[:32]
    print "    adding extra MD5 for ",n
    script.write('echo "'+md5+'  '+n+'" | md5sum -c > /dev/null\n')

  def patch_append(f):
    if VERBOSE > 2 :
      a=os.stat(TD+'PATCH/'+f)[ST_SIZE]
      print '   appending ',f,' of size ', a,' to debdelta, %3.2f'  % ( a * 100. /  newdebsize ) , '% of new .deb'
    system(['ar','qSc', delta,f],  TD+'/PATCH')
    unlink(TD+'PATCH/'+f)

  def verbatim(f):
    pp=a_numb_file.next()
    p = 'PATCH/'+pp
    if VERBOSE > 1 : print '  including "',name,'" verbatim in patch'
    os.rename(TD+f,TD+p)
    patch_append(pp)
    return p
      
  def unzip(f):
    c=''
    if f[-3:] == '.gz' :
      system('gunzip '+f,TD)
      f=f[:-3]
      c='.gz'
    elif  f[-4:] == '.bz2' :
      system('bunzip2 '+f,TD)
      f=f[:-4]
      c='.bz2'
    elif f[-5:] == '.lzma' :
      info_append('needs-lzma')
      system('unlzma '+f,TD)
      f=f[:-5]
      c='.lzma'
    else: raise NotImplementedError(' dont know how to decompress '+repr(f))
    return (f,c)

  def script_zip(n, cn,newhead=None):
    """inverts the unzip() function ; optionally, forces .gz header (to fight changes in libz)
    This is obsolete, not efficient, left as a compatibility layer."""
    script.write('cat "'+n+'" | ')
    script_zip_piped(cn, newhead)
    script.write(" > '"+n+cn+"' && rm '"+n+"'\n")

  def script_zip_piped(cn,newhead=None):
    "inverts the unzip() function, with piped.behaviour"
    if cn == '.gz' :
      if newhead:
        s=prepare_for_echo(newhead)
        script.write("($E '"+ s +"' && ./minigzip -9 | tail -c +"+str(len(newhead)+1)+')')
      else:
        script.write('./minigzip -9')
    elif  cn == '.bz2' :
      info_append('needs-minibzip2')
      script.write('./minibzip2 -9')
    elif cn == '.lzma' :
      info_append('needs-lzma')
      script.write('lzma -9')
    else: assert(0)

  def delta_files__(o,n,p,algo='bsdiff'):
    "delta of file 'o' to 'n' using/producing patch 'p' "
    #bdiff
    #http://www.webalice.it/g_pochini/bdiff/
    if algo == 'bdiff':
      system('~/debdelta/bdiff-1.0.5/bdiff -q -nooldmd5 -nonewmd5 -d  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+'\n')    
    #zdelta
    #http://cis.poly.edu/zdelta/
    elif algo == 'zdelta':
      system('~/debdelta/zdelta-2.1/zdc  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/zdelta-2.1/zdu '+o+' '+p+' '+n+'\n')
    #bdelta 
    #http://deltup.sf.net
    elif algo == 'bdelta':
      system('~/debdelta/bdelta-0.1.0/bdelta  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/bdelta-0.1.0/bpatch '+o+' '+n+' '+p+'\n')
    #diffball
    #http://developer.berlios.de/projects/diffball/
    elif algo == 'diffball':
      system('~/debdelta/diffball-0.7.2/differ  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/diffball-0.7.2/patcher '+o+' '+p+' '+n+'\n')
    #rdiff
    elif algo == 'rdiff':
      system('rdiff signature '+o+' sign_file.tmp  ',TD)
      system('rdiff delta  sign_file.tmp  '+n+' '+p,TD)
      script.write('rdiff patch '+o+' '+p+' '+n+'\n')
    #xdelta3
    elif algo == 'xdelta3' :
      system('xdelta3 -9 -R -D -n -S djw -s  '+o+' '+n+' '+p,TD,(o,n))
      script.write('xdelta3 -d -s '+o+' '+p+' '+n+'\n')
    ## according to the man page,
    ## bsdiff uses memory equal to 17 times the size of oldfile
    ## but , in my experiments, this number is more like 12.
    ##But bsdiff is sooooo slow!
    elif algo == 'bsdiff' : # not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)):    
      system('bsdiff  '+o+' '+n+' '+p,TD,(o,n))
      script.write('bspatch '+o+' '+n+' '+p+'\n')
    #seems that 'xdelta' is buggy on 64bit and different-endian machines
    #xdelta does not deal with different endianness!
    elif algo == 'xdelta-bzip' :
      system('xdelta delta --pristine --noverify -0 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD,(o,n))
      system('bzip2 -9 '+p,TD,(p,))
      script.write('bunzip2 '+p+'.bz2 ; xdelta patch '+p+' '+o+' '+n+'\n')
      p  += '.bz2'
    elif algo == 'xdelta' :
      system('xdelta delta --pristine --noverify -9 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD,(o,n))
      script.write('xdelta patch '+p+' '+o+' '+n+'\n')
    elif algo == 'jojodiff' :
      system('~/debdelta/jdiff06/src/jdiff -b '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/jdiff06/src/jpatch '+o+' '+p+' '+n+'\n')
    else: raise AssertionError(' unsupported delta algo ')
    return p

  def delta_files(o,n):
    " compute delta of two files , and prepare the script consequently"
    nsize = os.path.getsize(TD+n)
    osize = os.path.getsize(TD+o)
    if VERBOSE > 1 : print '  compute delta for %s (%dkB) and %s (%dkB)' % \
       (o,osize/1024,n,nsize/1024)
    #
    p = 'PATCH/'+a_numb_file.next()
    tim = -time.time()
    #
    if DEBUG > 3 :  script_md5_check_file(o)
    #
    if USE_DELTA_ALGO == 'bsdiff' and osize > ( 1.1 * (MAXMEMORY / 12))  and VERBOSE  :
      print ' Warning, memory usage by bsdiff on the order of %dMb' % (12 * osize / 2**20)
    #
    p = delta_files__(o,n,p,USE_DELTA_ALGO)
    #script.write(s)
    #
    if DEBUG > 2 :  script_md5_check_file(n)
    #
    tim += time.time()      
    #
    global bsdiff_time, bsdiff_datasize
    bsdiff_time += tim
    bsdiff_datasize += nsize
    #
    script.write('rm '+o+' '+p+'\n')
    ## how did we fare ?
    deltasize = os.path.getsize(TD+p)
    if VERBOSE > 1 :
      print '  delta is %3.2f%% of %s, speed: %dkB /sec'  % \
          ( ( deltasize * 100. /  nsize ) , n, (nsize / 1024. / ( tim + 0.001 )))
    #possibly GPG
    if DO_GPG:
      gpg_hashes[p[6:]] = _compute_hashes_(TD+p)
    #save it
    patch_append(p[6:])
    #clean up
    unlink(TD+o)

  def cmp_gz(o,n):
    "compare gzip files, ignoring header; returns first different byte (+-10), or True if equal"
    of=open(TD+o)
    nf=open(TD+n)
    oa=of.read(10)
    na=nf.read(10)
    if na[:3] != '\037\213\010' :
      print ' Warning: was not created with gzip: ',n
      nf.close() ; of.close() 
      return 0
    if oa[:3] != '\037\213\010' :
      print ' Warning: was not created with gzip: ',o
      nf.close() ; of.close() 
      return 0
    oflag=ord(oa[3])
    if oflag & 0xf7:
      print ' Warning: unsupported  .gz flags: ',oct(oflag),o
    if oflag & 8 : #skip orig name
      oa=of.read(1)
      while ord(oa) != 0:
        oa=of.read(1)
    l=10
    nflag=ord(na[3])
    if nflag & 0xf7:
      print ' Warning: unsupported  .gz flags: ',oct(nflag),n
    if nflag & 8 : #skip orig name
      na=nf.read(1)
      s=na
      while ord(na) != 0:
        na=nf.read(1)
        s+=na
      l+=len(s)
      #print repr(s)
    while oa and na:
      oa=of.read(2)
      na=nf.read(2)
      if oa != na:
        return l
      l+=2
    if oa or na: return l
    return True
    
  def delta_gzipped_files(o,n):
    "delta o and n, replace o with n"
    assert(o[-3:] == '.gz' and n[-3:] == '.gz')
    before=cmp_gz(o,n)
    if before == True:
      if VERBOSE > 3: print '    equal but for header: ',n
      return
    #compare the cost of leaving as is , VS the minimum cost of delta
    newsize=os.path.getsize(TD+n)
    if ( newsize - before + 10 ) < 200 :
      if VERBOSE > 3: print '    not worthwhile gunzipping: ',n
      return
    f=open(TD+n)
    a=f.read(10)
    f.close()
    if a[:3] != '\037\213\010' :
      print ' Warning: was not created with gzip: ',n
      return
    flag=ord(a[3]) # mostly ignored  :->
    orig_name='-n'
    if flag & 8:
      orig_name='-N'
    if flag & 0xf7:
      print ' Warning: unsupported  .gz flags: ',oct(flag),n
    #a[4:8] #mtime ! ignored ! FIXME will be changed... 
    #from deflate.c in gzip source code
    format=ord(a[8])
    FAST=4
    SLOW=2 #unfortunately intermediate steps are lost....
    pack_level=6
    if format ==  0 :
      pass
    elif format ==  FAST :
      pack_level == 1
    elif format ==  SLOW :
      pack_level == 9
    else:
      print ' Warning: unsupported compression .gz format: ',oct(format),n
      return
    if a[9] != '\003' :
      if VERBOSE : print ' Warning: unknown OS in .gz format: ',oct(ord(a[9])),n
    p='_tmp_'
    #save new file and unzip
    shutil.copy2(TD+n,TD+p+'.new.gz')
    system("gunzip '"+n+"'",TD)
    shutil.copy2(TD+n[:-3],TD+p+'.new')
    #test our ability of recompressing
    l=[1,2,3,4,5,6,7,8,9]
    del l[pack_level]
    l.append(pack_level)
    l.reverse()
    for i in l:
      #force -n  ... no problem with timestamps
      gzip_flags="-n -"+str(i)      
      system("gzip -c "+gzip_flags+" '"+n[:-3]+"' > "+p+'.faked.gz',TD)
      r=cmp_gz(p+'.new.gz',p+'.faked.gz')
      if r == True:
        break
      if i == pack_level and VERBOSE > 3:
        print '    warning: wrong guess to re-gzip to equal file: ',gzip_flags,r,n
    if r != True:
      if VERBOSE > 2 : print '   warning: cannot re-gzip to equal file: ',r,n
      os.unlink(TD+p+".new") ; os.unlink(TD+p+'.new.gz') ; os.unlink(TD+p+'.faked.gz') 
      return
    #actual delta of decompressed files
    system("zcat '"+o+"' > "+p+'.old',TD)
    script.write("zcat '"+o+"' > "+p+".old ; rm '"+o+"' \n")
    if VERBOSE > 2 : print '   ',n[9:],'  (= to %d%%): ' % (100*before/newsize) ,
    delta_files(p+'.old',p+'.new')
    os.rename(TD+p+'.faked.gz',TD+o)
    script.write("gzip -c "+gzip_flags+" < "+p+".new  > '"+o+"' ; rm "+p+".new\n")
    if DEBUG > 1 :  script_md5_check_file(o)
    os.unlink(TD+p+'.new.gz')
    
  ########### helper sh functions for script, for delta_tar()

  import difflib

  def file_similarity_premangle(oo):
    o=oo.split('/')
    (ob,oe)=os.path.splitext(o[-1])
    return o[:-1]+ ob.split('_')+[oe]
  
  def files_similarity_score__noext__(oo,nn):
    ln=len(nn)
    lo=len(oo)
    l=0
    while oo and nn:
      while oo and nn and oo[-1] == nn[-1]:
        oo=oo[:-1]
        nn=nn[:-1]
      if not oo or not nn: break
      while oo and nn and oo[0] == nn[0]:
        oo=oo[1:]
        nn=nn[1:]
      if not oo or not nn: break
      if len(nn) > 1 and oo[0] == nn[1]:
        l+=1
        nn=nn[1:]
      if len(oo) > 1 and oo[1] == nn[0]:
        l+=1
        oo=oo[1:]
      if not oo or not nn: break
      if  oo[-1] != nn[-1]:
        oo=oo[:-1]
        nn=nn[:-1]
        l+=2
      if not oo or not nn: break
      if oo[0] != nn[0]:
        oo=oo[1:]
        nn=nn[1:]
        l+=2
    return (l +len(oo) + len(nn)) * 2.0 / float(ln+lo)

  def files_similarity_score__(oo,nn):
    oo=copy(oo)
    nn=copy(nn)
    if oo.pop() != nn.pop() :
      return 0.2 + files_similarity_score__noext__(oo,nn)
    else:
      return files_similarity_score__noext__(oo,nn)
  
  def files_similarity_score__difflib__(oo,nn):
    "compute similarity by difflib. Too slow."
    if oo == nn :
      return 0
    d=difflib.context_diff(oo,nn,'','','','',0,'')
    d=[a for a in tuple(d) if a and a[:3] != '---' and a[:3] != '***' ]
    if oo[-1] != nn[-1] : #penalty for wrong extension
      return 0.2+float(len(d)) * 2.0 / float(len(oo)+len(nn))
    else:
      return float(len(d)) * 2.0 / float(len(oo)+len(nn))
    
  def files_similarity_score(oo,nn):
    if oo == nn :
      return 0
    if type(oo) == StringType:
      oo=file_similarity_premangle(oo)
    if type(nn) == StringType:
      nn=file_similarity_premangle(nn)
    return files_similarity_score__(oo,nn)

  def fake_tar_header_2nd():
    " returns the second part of a tar header , for regular files and dirs"
    # The following code was contributed by Detlef Lannert.
    # into /usr/lib/python2.3/tarfile.py
    MAGIC      = "ustar"            # magic tar string
    VERSION    = "00"               # version number
    NUL        = "\0"               # the null character
    parts = []
    for value, fieldsize in (
      ("", 100),
      # unfortunately this is not what DPKG does
      #(MAGIC, 6),
      #(VERSION, 2),
      #  this is  what DPKG does
      ('ustar  \x00',8),
      ("root", 32),
      ("root", 32),
      ("%07o" % 0, 8),
      ("%07o" % 0, 8),
      ("", 155)
      ):
      l = len(value)
      parts.append(value + (fieldsize - l) * NUL)      
    buf = "".join(parts)
    return buf
  
  fake_tar_2nd=fake_tar_header_2nd()
  fake_tar_2nd_echo=prepare_for_echo(fake_tar_2nd)
  script.write("FTH='"+fake_tar_2nd_echo+"'\n")
  script.write("E='echo -ne'\n")
  
  script.write('CR () { cat "$1"  >> OLD/mega_cat ; rm "$1" ;}\n')
  
  global time_corr
  time_corr=0

  ####################  vvv     delta_tar    vvv ###########################
  def delta_tar(old_filename,new_filename,CWD,skip=[],old_md5={},new_md5={}, chunked_p=True,debdelta_conf_skip=()):
    " compute delta of two tar files, and prepare the script consequently"
    assert( type(old_filename) == StringType or type(old_filename) == FunctionType )
    if os.path.exists(TD+'OLD/mega_cat'):
      print 'Warning!!! OLD/mega_cat  exists !!!!'
      # if -k is given, still we need to delete it...
      os.unlink(TD+'OLD/mega_cat')
      script.write('rm OLD/mega_cat || true \n')
    mega_cat=open(TD+'OLD/mega_cat','w')
    #helper function
    def _append_(w,rm=False):
      assert(os.path.isfile(TD+w))
      f=open(TD+w)
      a=f.read(1024)
      while a:
        try:
          mega_cat.write(a)
        except OSError,s :
          raise DebDeltaError(' OSError (at _a_) while writing: '+str(s), True)
        a=f.read(1024)
      f.close()
      if rm:
        script.write("CR '"+w+"'\n")
        unlink(TD+w)
      else:
        script.write("cat '"+w+"'  >> OLD/mega_cat\n")

    #### scan once for regular files
    if type(old_filename) == StringType :
      (old_filename,old_filename_ext) = unzip(old_filename)
      oldtar = tarfile.open(TD+old_filename, "r")
    else:
      old_filename_ext=None
      oldfileobj = old_filename()
      oldtar = tarfile.open(mode="r|", fileobj=oldfileobj)
    oldnames = []
    oldtarinfos = {}
    for oldtarinfo in oldtar:
      oldname = de_bar(oldtarinfo.name)

      #this always happens
      #if VERBOSE > 3 and oldname != de_bar(oldname):
      #  print '     filename in old tar has weird ./ in front: ' , oldname 

      if  not oldtarinfo.isreg():
        if VERBOSE > 2 : print '  skipping old non-regular ',repr(oldname)
        continue

      if  oldtarinfo.size == 0:
        if VERBOSE > 2 : print '  skipping old empty ',repr(oldname)
        continue

      if shell_not_allowed(oldname):
        if VERBOSE > 2 : print '  skipping non-allowed-name ',repr(oldname)
        continue

      for j in debdelta_conf_skip:
        if fnmatch.fnmatch(oldname,j):
          if VERBOSE > 2 : print '  skipping following as per rule ',repr(j)
          skip.append(oldname)
          break
      
      if oldname in skip:
        if VERBOSE > 2 : print '  skipping ',repr(oldname)
        continue

      oldnames.append(oldname)
      oldtarinfos[oldname] = oldtarinfo
      oldtar.extract(oldtarinfo,TD+"OLD/"+CWD )
    oldtar.close()
    if type(old_filename) == StringType :
      unlink(TD+old_filename)
    else:
      while oldfileobj.read(512):
        pass
    #save header part of new_filename, since it changes in newer versions
    f=open(TD+new_filename)
    new_file_zip_head=f.read(20)
    f.close()
    (new_filename,new_filename_ext) = unzip(new_filename)
    assert(0 == (os.path.getsize(TD+new_filename)% 512))
    newtar = tarfile.open(TD+new_filename, "r")
    newnames = []
    newtarinfos = {}
    for newtarinfo in newtar:
      newname =  newtarinfo.name
      #just curious to know
      t=newtarinfo.type
      a=newtarinfo.mode
      if VERBOSE and (( t == '2' and a  != 0777 ) or \
                      ( t == '0' and ( (a & 0400 ) == 0 )) or \
                      ( t == '5' and ( (a & 0500 ) == 0 ))):
        print ' weird permission: ',newname,oct(a),repr(newtarinfo.type)
      ###
      if   not newtarinfo.isreg():
        continue
      if VERBOSE > 3 and newname != de_bar(newname):
        print '    filename in new tar has weird ./ in front: ' , newname 
      newname = de_bar(newname)
      newnames.append(newname)
      newtarinfos[newname] = newtarinfo
      
    old_used={}
    correspondence={}

    ##############################
    global time_corr
    time_corr=-time.time()

    if VERBOSE > 2 : print '  finding correspondences for ', new_filename

    reverse_old_md5={}
    if old_md5:
      for o in old_md5:
        if o in oldnames:
          reverse_old_md5[old_md5[o]] = o
        else:
          #would you believe? many packages contain MD5 for files they do not ship...
          if VERBOSE > 1 and o not in skip: print '  hmmm... there is a md5 but not a file: ',o

    oldnames_premangle={}
    for o in oldnames:
      a,b=os.path.splitext(o)
      if b not in oldnames_premangle:
        oldnames_premangle[b]={}
      oldnames_premangle[b][o]=file_similarity_premangle(a)

    for newname in newnames:
      newtarinfo=newtarinfos[newname]
      oldname=None
      #ignore empty files
      if newtarinfo.size == 0:
        continue
      #try correspondence by MD5
      if new_md5 and newname in new_md5:
        md5=new_md5[newname]        
        if md5 in reverse_old_md5:
          oldname=reverse_old_md5[md5]
          if VERBOSE > 2 :
            if oldname  == newname :
              print '   use identical old file: ',newname
            else:
              print '   use identical old file: ',oldname, newname
      #try correspondence by file name
      if oldname == None and newname in oldnames:
        oldname=newname
        if VERBOSE > 2 : print '   use same name old file: ',newname
      #try correspondence by file name and len similarity
      nb,ne=os.path.splitext(newname)
      if oldname == None and ne in oldnames_premangle:
        basescore=1.6
        nl=newtarinfo.size
        np=file_similarity_premangle(nb)
        for o in oldnames_premangle[ne]:
          op=oldnames_premangle[ne][o]
          l=oldtarinfos[o].size
          sfile=files_similarity_score__noext__(op,np)
          slen = abs(float(l - nl))/float(l+nl)
          s=slen+sfile
          if VERBOSE > 3 : print '    name/len diff %.2f+%.2f=%.2f ' % (slen,sfile,s), o
          if s < basescore:
              oldname=o
              basescore=s
        if oldname and VERBOSE > 2 : print '   best similar  ','%.3f' % basescore,newname,oldname
      if not oldname:
        if VERBOSE > 2 : print '   no correspondence for: ',newname
        continue
      #we have correspondence, lets store
      if oldname not in old_used:
        old_used[oldname]=[]
      old_used[oldname].append(newname)
      correspondence[newname]=oldname
      
    time_corr+=time.time()
    if VERBOSE > 1 : print '  time lost so far in finding correspondence %.2f' % time_corr
    
    ######### now do real scanning
    if VERBOSE > 2 : print '  scanning ',new_filename

    #helper function
    def mega_cat_chunk(oldoffset,newoffset):
      p = a_numb_file.next()
      f=open(TD+new_filename)
      f.seek(oldoffset)
      of=open(TD+p,'w')
      l=oldoffset
      while l<newoffset:
        s=f.read(512)
        l+=len(s)
        assert(len(s))
        try:
          of.write(s)
        except OSError,s :
          raise DebDeltaError(' OSError (at MCK) while writing: '+str(s), True)
      f.close()
      of.close()
      #move to a temporary
      pt=a_numb_file.next()
      script.write('mv OLD/mega_cat '+pt+'\n')
      os.rename(TD+'OLD/mega_cat',TD+pt)
      #do delta, in background there
      script.write('wait ; ( ')
      delta_files(pt,p)
      script.write('cat '+p+' >> '+new_filename+'; rm '+p+' ; ) & \n')
      os.unlink(TD+p)

    #there may be files that have been renamed and edited...
    def some_old_file_gen():
      for oldname in oldnames :
        if (oldname in skip) or (oldname in old_used ) :
          continue
        if VERBOSE > 2 : print '   provide also old file ', oldname
        yield oldname
      while 1:
        yield None

    some_old_file=some_old_file_gen()
    one_old_file=some_old_file.next()

    max_chunk_size = MAXMEMORY / 12
    chunk_discount = 0.3

    progressive_new_offset=0

    for newtarinfo in newtar:
      ## for tracking strange bugs
      if DEBUG > 3 and mega_cat.tell() > 0 :
        script_md5_check_file("OLD/mega_cat")
      #progressive mega_cat
      a=mega_cat.tell()
      if chunked_p and ((a >=  max_chunk_size * chunk_discount) or \
         (a >= max_chunk_size * chunk_discount * 0.9 and one_old_file ) or \
         (a>0 and (a+newtarinfo.size) >= max_chunk_size * chunk_discount )):
        #provide some old unused files, if any
        while one_old_file:
          w="OLD/"+CWD+"/"+one_old_file
          if os.path.isfile(TD+w):
            _append_(w)
          else: print 'Warning!!! ',w,'does not exists ???'
          if mega_cat.tell() >=  max_chunk_size * chunk_discount :
            break
          one_old_file=some_old_file.next()
        mega_cat.close()
        mega_cat_chunk(progressive_new_offset, newtarinfo.offset )
        progressive_new_offset=newtarinfo.offset
        mega_cat=open(TD+'OLD/mega_cat','w')
        chunk_discount = min( 1. , chunk_discount * 1.2 )
      #
      name = de_bar( newtarinfo.name )
      #recreate also parts of the tar headers
      mega_cat.write(newtarinfo.name+fake_tar_2nd)
      s=prepare_for_echo(newtarinfo.name)
      script.write("$E '"+ s +"'\"${FTH}\" >> OLD/mega_cat\n")

      if newtarinfo.isdir():
        if VERBOSE > 2 : print '   directory   in new : ', name
        continue

      if not newtarinfo.isreg():
        if VERBOSE > 2 : print '   not regular in new : ', name
        continue

      if newtarinfo.size == 0:
        if VERBOSE > 2 : print '   empty  new file    : ', name
        continue

      if name not in correspondence:
        if VERBOSE > 2: print '   no corresponding fil: ', name
        continue 
      oldname = correspondence[name]

      mul=len( old_used[oldname]) > 1 #multiple usage
      
      if not mul and oldname == name and oldname[-3:] == '.gz' and \
             newtarinfo.size > 120 and  \
        not ( new_md5 and name in new_md5 and old_md5 and name in old_md5 and \
           new_md5[name] == old_md5[name]):
        newtar.extract(newtarinfo,TD+"NEW/"+CWD )
        delta_gzipped_files("OLD/"+CWD+'/'+name,"NEW/"+CWD+'/'+name)

      if VERBOSE > 2 :  print '   adding reg file: ', oldname, mul and '(multiple)' or ''
      _append_( "OLD/"+CWD+"/"+oldname , not mul )
      old_used[oldname].pop()


    mega_cat.close()
    if os.path.exists(TD+'/OLD/'+CWD):
      rmtree(TD+'/OLD/'+CWD)
    if os.path.getsize(TD+'OLD/mega_cat') > 0 :
      if progressive_new_offset > 0 :
        assert(chunked_p)
        mega_cat_chunk(progressive_new_offset, os.path.getsize(TD+new_filename))
      else:
        delta_files('OLD/mega_cat',new_filename)
        unlink(TD+new_filename)
    else:
      p=verbatim(new_filename)
      script.write('mv '+p+' '+new_filename+ '\n')
    script.write('wait\n')
    script_zip(new_filename,new_filename_ext,new_file_zip_head)
  ####################  ^^^^    delta_tar    ^^^^ ###########################

  ############ start computing deltas  
  def append_NEW_file(s):
    'appends some data to NEW.file'
    s=prepare_for_echo(s)
    script.write("$E '"+ s +"' >> NEW.file\n")
    
  #this following is actually
  #def delta_debs_using_old(old,new):

  ### start scanning the new deb  
  newdeb_file=open(newdeb)
  # pop the "!<arch>\n"
  s = newdeb_file.readline()
  assert( "!<arch>\n" == s)
  append_NEW_file(s)

  #process all contents of old vs new .deb
  ar_list_old= list_ar(TD+'OLD.file')
  ar_list_new= list_ar(TD+'NEW.file')

  def md5_ar(TD,n,name):
    "extra md5 check, for tracking strange bugs"
    pm=my_popen_read('cd '+TD+'; ar p OLD.file '+name+' | md5sum -')
    data_tar_md5=pm.readline()[:32]
    pm.read()
    pm.close()
    script_md5_check_file(n,data_tar_md5)

  for name in ar_list_new :
    newname = 'NEW/'+name
    system('ar p '+TD+'NEW.file '+name+' >> '+TD+newname,TD)

    newsize = os.stat(TD+newname)[ST_SIZE]
    if VERBOSE > 1: print '  studying ' , name , ' of len %dkB' % (newsize/1024)
    #add 'ar' structure
    s = newdeb_file.read(60)
    if VERBOSE > 3: print '    ar line: ',repr(s)
    assert( s[:len(name)] == name and s[-2] == '`' and s[-1] == '\n' )
    append_NEW_file(s)
    #sometimes there is an extra \n, depending if the previous was odd length
    newdeb_file.seek(newsize  ,1)
    if newsize & 1 :
      extrachar = newdeb_file.read(1)
    else:
      extrachar = ''
    #add file to debdelta
    if newsize < 128:      #file is too short to compute a delta,
      p=open(TD+newname)
      append_NEW_file( p.read(newsize))
      p.close()
      unlink(TD+newname)
    elif not NEEDSOLD and name[:11] == 'control.tar' :
      #(mm this is almost useless, just saves a few bytes)
      oldname = 'OLD/'+name
      system('ar p OLD.file '+name+' >> '+oldname, TD)
      ##avoid using strange files that dpkg may not install in /var...info/
      skip=[]
      for a in os.listdir(TD+'OLD/CONTROL') :
        if a not in dpkg_keeps_controls:
          skip.append(a)
      #delta it
      #never chunked .. otherwise the first file in the ar will not be '0'!
      delta_tar(oldname,newname,'CONTROL',skip, chunked_p=False)
      if DEBUG > 3 : md5_ar(TD,newname,name)
      script.write('cat '+newname+' >> NEW.file ;  rm '+newname+'\n')
    elif not NEEDSOLD and name[:8] == 'data.tar'  :
      if 'data.tar.gz' in ar_list_old  :
        def x():
          return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.gz | gzip -cd')
      elif 'data.tar.bz2' in ar_list_old :
        def x():
          return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.bz2 | bzip2 -cd')
      elif 'data.tar.lzma' in ar_list_old :
        info_append('needs-lzma')
        def x():
          return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.lzma | unlzma -c')
      else: assert(0)
      delta_tar(x,newname,'DATA',old_conffiles,old_md5,new_md5,\
                debdelta_conf_skip=debdelta_conf_skip)
      del x
      if DEBUG > 3 : md5_ar(TD,newname,name)
      script.write('cat '+newname+' >> NEW.file ;  rm '+newname+'\n')
    elif  not NEEDSOLD  or name not in ar_list_old :   #or it is not in old deb
      patchname=verbatim(newname)
      script.write('cat '+patchname+' >> NEW.file ; rm '+patchname+'\n')
    elif  NEEDSOLD :
      #file is long, and has old version ; lets compute a delta
      oldname = 'OLD/'+name
      system('ar p OLD.file '+name+' >> '+oldname, TD)
      script.write('ar p OLD.file '+name+' >> '+oldname+'\n')
      (oldname,co) = unzip(oldname)
      (newname,cn) = unzip(newname)
      delta_files(oldname,newname)
      script_zip(newname,cn)
      script.write('cat '+newname+cn+' >> NEW.file ;  rm '+newname+cn+'\n')
      unlink(TD+newname)
      del co,cn
    else:
      die('internal error j98')
    #pad new deb
    if extrachar :
      append_NEW_file(extrachar)
  # put in script any leftover
  s = newdeb_file.read()
  if s:
    if VERBOSE > 2: print '   ar leftover character: ',repr(s)
    append_NEW_file(s)
  del s

  #this is done already from the receiving end
  if DEBUG > 2 and newdeb_md5sum :
    script_md5_check_file("NEW.file",md5=newdeb_md5sum)
  
  #script is done
  script.close()

  patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE]
  v=''
  #if VERBOSE > 1 :v ='-v' #disabled... it does not look good inlogs
  patch_files = []
  if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'):
    system('lzma -q -9 -k '+v+' PATCH/patch.sh', TD)
    patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.lzma'), 'lzma', 'patch.sh.lzma'))
  system('bzip2 -q --keep -9  '+v+'  PATCH/patch.sh', TD)
  patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.bz2'), 'bzip2', 'patch.sh.bz2'))
  system('gzip -q -9 -n '+v+' PATCH/patch.sh', TD)
  patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.gz'), 'gzip', 'patch.sh.gz'))
  del v

  # Use the smallest compressed patch.sh
  patch_files.sort()
  if VERBOSE > 1 : print '  '+patch_files[0][1]+' wins on patch.sh'
  if patch_files[0][1] == 'lzma':
    info_append('needs-lzma')
  
  if DO_GPG:
    gpg_hashes[patch_files[0][2]] = _compute_hashes_(TD+"PATCH/"+patch_files[0][2])
  
  patch_append(patch_files[0][2])
  del patch_files
  
  #OK, OK... this is not yet correct, since I will add the info file later on
  elaps =  time.time() - start_sec
  info.append('DeltaTime: %.2f' % elaps)
  deltasize = os.stat(delta)[ST_SIZE] + 60 + sum(map(len,info))
  percent =  deltasize * 100. /  newdebsize
  info.append('Ratio: %.4f' % (float(deltasize) / float(newdebsize)) )

  if VERBOSE:
    print ' deb delta is  %3.1f%% of deb; that is, %dkB are saved, on a total of %dkB.' \
          % ( percent , (( newdebsize -deltasize ) / 1024),( newdebsize/ 1024))
    print ' delta time: %.2f sec, speed: %dkB /sec, (%s time: %.2fsec speed  %dkB /sec) (corr %.2f sec)' %  \
          (elaps, newdebsize / 1024. / (elaps+0.001), \
           USE_DELTA_ALGO,bsdiff_time, bsdiff_datasize / 1024. / (bsdiff_time + 0.001) , time_corr )
  return (delta, percent, elaps, info, gpg_hashes)


##################################################### compute many deltas

def do_deltas(debs):
  exitstatus=0
  
  original_cwd = os.getcwd()
  start_time = time.time()
  import warnings
  warnings.simplefilter("ignore",FutureWarning)
  try:
    from apt import VersionCompare
  except ImportError:
    try:
      import apt_pkg
      apt_pkg.InitSystem()
      from apt_pkg import VersionCompare
    except ImportError:
      raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt', retriable=True)
    
  f=my_popen_read('hostname -f')
  try:
    import hashlib
    HOSTID=hashlib.md5( f.read() ).hexdigest()
  except ImportError:
    import md5
    HOSTID=md5.new( f.read() ).hexdigest()
  f.close()
    
  if AVOID and type(AVOID) == StringType:
    import shelve
    if VERBOSE : print ' Using avoid dict ',AVOID
    avoid_pack = shelve.open(AVOID,'r')
  else:
    avoid_pack = {}
  
  info_by_pack_arch={}
  info_by_file={}
  
  def info_by_pack_arch_add(f):
    pack = info_by_file[f]['Package']
    arch = info_by_file[f]['Architecture']
    vers = info_by_file[f]['Version']
    if pack in avoid_pack and ( avoid_pack[pack]['Version'] == vers ):
      #note that 'f' is in  info_by_file and not in info_by_pack_arch
      if VERBOSE > 1 :     print '  Avoid: ', f
      return
    if  (pack,arch) not in  info_by_pack_arch :
      info_by_pack_arch[ (pack,arch) ]=[]
    info_by_pack_arch[ (pack,arch) ].append( info_by_file[f] )
    
  def iterate_Packages(packages):
    packages=abspath(packages)
    assert os.path.isfile(packages)
    assert os.path.basename(packages) in  ('Packages', 'Packages.gz','Packages.bz2')
    dir=os.path.dirname(packages)
    dir=dir.split('/')
    try:
      a=dir.index('dists')
    except ValueError:
      sys.stderr.write('Error: pathname "%s" does not contain "dists"\n' % packages)
      return
    base = string.join(dir[:a],'/')
    if packages[-3:] == '.gz':
      F=subprocess.Popen(["zcat",packages],stdout=subprocess.PIPE).stdout
    elif packages[-4:] == '.bz2':
      F=subprocess.Popen(["bzcat",packages],stdout=subprocess.PIPE).stdout
    else:
      F=open(packages)
    for l in F:
      l=l.rstrip('\n')
      if l[:9] == 'Package: ':
          pack=l[9:]
      elif l[:14] == 'Architecture: ':
          arch = l[14:]
      elif l[:9] ==  'Version: ':
          vers = l[9:]
      elif l[:10] == 'Filename: ':
            of=l[10:]
      elif l == '':
        f=base+'/'+of
        if f[-4:] == '.udeb':
          if VERBOSE > 2 : print '   skip udeb: ',f
          continue
        elif not os.path.isfile(f):
          sys.stderr.write('Package missing! '+f+'\n')
          continue
        if f not in info_by_file:
          info_by_file[f]={}
          info_by_file[f]['File'] = f
          info_by_file[f]['Filename'] = of
          info_by_file[f]['Package']  = pack
          info_by_file[f]['Architecture'] = arch
          info_by_file[f]['Version'] = vers
        else:  #this happens e.g. if the package is both in testing and in unstable..
          if VERBOSE > 2 : print '   seen twice, do not re-add info: ',f
          assert info_by_file[f]['Version'] == vers and \
                 info_by_file[f]['Package'] == pack and \
                 info_by_file[f]['Architecture'] == arch
        yield f
        del of,pack,vers,arch
        
  def scan_Packages(packages, label):
    for f in iterate_Packages(packages):
      info_by_file[f]['Label'] = label
      info_by_pack_arch_add(f)
  
  deb_dir_cache={}
  delta_dir_cache={}
  def cache_dir(f,extension='.deb',dacache=deb_dir_cache):
    assert( os.path.isdir(f))
    if f in dacache:
      return dacache[f]
    cache={}
    for d in os.listdir(f):
      a,b = os.path.splitext(d)
      if b == extension:
        a=os.path.join(f,d)
        n=d.split('_')[0]
        if n not in cache:
          cache[n]=[a]
        else:
          cache[n].append(a)
    dacache[f] = cache
    return cache
  
  def scan_deb_dir(f, debname, label):
    assert os.path.isdir(f) and (debname == None or type(debname) == StringType) and type(label) == StringType
    cache = cache_dir(f)
    if debname == None:
      for n in cache:
        for dt in cache[n]:
          scan_deb( dt , label )
    else:
      if debname in cache:
        for dt in cache[debname]:
          scan_deb( dt , label )

  def scan_deb(of, label):
      assert( os.path.isfile(of) )
      f=abspath(of)
      if f in info_by_file:
        #just (in case) promote to status of CMDLINE package
        if label == 'CMDLINE' and info_by_file[f]['Label'] !=  'CMDLINE':
          if VERBOSE > 2 : print '   promoting to CMDLINE ',f
          #this changes also the entry in info_by_pack_arch (magic python)
          info_by_file[f]['Label']=label
        return
      p=open(f)
      if p.read(21) != "!<arch>\ndebian-binary" :
        p.close()
        if os.path.getsize(f) == 0 :
          print ('Warning: '+f+ ' is an empty file; removing it. ')
          if ACT : os.unlink(f)
        else:  
          print ('Error: '+f+ ' does not seem to be a Debian package ')
        return
      p.close()
      info_by_file[f]={}
      p=my_popen_read('ar p '+f+' control.tar.gz | tar -x -z -f - -O ./control')
      scan_control(p,params=info_by_file[f])
      p.close()
      info_by_file[f]['Filename'] = of
      info_by_file[f]['File'] = f
      info_by_file[f]['Label'] = label
      info_by_pack_arch_add(f)

  def scan_delta_dir(f,debname=None):
    if not os.path.isdir(f) :
      if VERBOSE > 1 : print '  no such delta dir: ',f
      return
    assert debname == None or type(debname) == StringType 
    cache = cache_dir(f,extension='.delta',dacache=delta_dir_cache)
    if debname == None :
      for n in cache:
        for dt in cache[n]:
          scan_delta( dt )
    else:
      if debname in cache:
        for dt in cache[debname]:
          scan_delta( dt )
    
  # contains list of triples (filename,oldversion,newversion)
  old_deltas_by_pack_arch={}
  
  def scan_delta(f):
    assert( os.path.isfile(f) )
    if f[-9:] == '.debdelta' :
      a=f[:-9]
    elif f[-17:] == '.debdelta-too-big' :
      a=f[:-17]
    elif f[-15:] == '.debdelta-fails' :
      a=f[:-15]
    else: return
    a=os.path.basename(a)
    a=a.split('_')
    pa=a[0]
    ar=a[3]
    if  (pa,ar) not in old_deltas_by_pack_arch:
      old_deltas_by_pack_arch[ (pa,ar) ]=[]
    ov=version_demangle(a[1])
    nv=version_demangle(a[2])
    if (f,ov,nv) not in old_deltas_by_pack_arch[ (pa,ar) ]:
      old_deltas_by_pack_arch[ (pa,ar) ].append( (f, ov, nv ) )

  def delta_dirname(f,altdir):
    "compute augmented dirname"
    if os.path.isfile(f):
      f=os.path.dirname(f) or '.'
    assert(os.path.isdir(f))
    if altdir:
      if altdir[-2:] == '//' :
        a=altdir+f
        return abspath(a)+'/'
      else:
        return altdir
    else:
      return abspath(f)

  def package_name(n):
    "returns the package name from the file name"
    n=os.path.basename(n)
    n=n.split('_')[0] 
    return n
  
  #reduce ALT, by preprocessing Packages
  ALT_NOP=[]
  for alt in ALT:
    if os.path.basename(alt) in ('Packages', 'Packages.gz','Packages.bz2'):
      scan_Packages(alt,'ALT')
    else:
      ALT_NOP.append(alt)

  #scan cmdline arguments and prepare list of debs and deltas
  for arg in debs:
    if os.path.isfile(arg):
      if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'):
        for a in  iterate_Packages(arg):
          info_by_file[a]['Label'] = 'CMDLINE'
          of = info_by_file[a]['Filename']
          info_by_pack_arch_add(a)
          pa = package_name(a)
          for alt in ALT_NOP:
            scan_deb_dir(delta_dirname(of,alt), pa , 'ALT' )
          if CLEAN_DELTAS:
            scan_delta_dir(delta_dirname(of,DIR), pa )
      elif arg[-4: ] != '.deb' :
        print 'Warning: skipping cmd line argument: ',arg
      else:
        scan_deb(arg, 'CMDLINE')
        di = os.path.dirname(arg) or '.'
        pa = package_name(arg)
        scan_deb_dir(di, pa, 'SAMEDIR' )
        for alt in ALT_NOP:
          scan_deb_dir(delta_dirname(arg,alt), pa, 'ALT')
        if CLEAN_DELTAS:
          scan_delta_dir(delta_dirname(arg,DIR), pa)
    elif  os.path.isdir(arg):
      scan_deb_dir(arg, None, 'CMDLINE')
      for alt in ALT_NOP:
        if alt[-2:] == '//':
          scan_deb_dir(delta_dirname(arg,alt), None, 'ALT')
      if CLEAN_DELTAS:
        scan_delta_dir(delta_dirname(arg,DIR))
    else:
      print 'Warning: '+arg+' is not a regular file or a directory.'

  if VERBOSE > 1 : print '  total parsing time: %.1f ' % ( -start_time + time.time())
  
  def order_by_version(a,b):
    return VersionCompare( a['Version'] , b['Version']  )
  
  for pa,ar in info_by_pack_arch :
    info_pack=info_by_pack_arch[ (pa,ar) ]
    info_pack.sort(order_by_version)

    versions = [ o['Version'] for o in info_pack ]

    versions_not_alt = [ o['Version'] for o in info_pack if o['Label'] != "ALT" ]

    #delete deltas that are useless
    if CLEAN_DELTAS and (pa,ar) in old_deltas_by_pack_arch :
      for f_d,o_d,n_d in old_deltas_by_pack_arch[ (pa,ar) ] :
        if n_d not in versions_not_alt :
          if os.path.exists(f_d):
            if VERBOSE: print ' removing: ',f_d          
            if ACT: os.unlink(f_d)
    
    how_many= len(info_pack)
    if VERBOSE > 2 : print '   Package: ',pa,' Versions:',versions
    if how_many <= 1 :
      continue
    
    newest = how_many -1
    while newest >= 0 :
      new=info_pack[newest]
      if new['Label'] != 'CMDLINE' :
        if VERBOSE > 1 : print '  Newest version deb was not in cmdline, skip down one: ', new['File']
      else:
        break
      newest -= 1

    if newest <= 0 :
      continue

    newdebsize=os.path.getsize(new['File'])
    #very small packages cannot be effectively delta-ed
    if newdebsize <= MIN_DEB_SIZE :
      if VERBOSE > 1 : print '  Skip , too small: ', new['File']
      continue

    l = newest
    while (l>0) and (l > newest - N_DELTAS):
        l -= 1
        old=info_pack[l]
        
        if  old['Version'] == new['Version'] :
          continue
                
        assert( old['Package'] == pa and pa == new['Package'] )
        deltabasename = pa +'_'+  version_mangle(old['Version']) +\
                        '_'+ version_mangle(new['Version']) +'_'+ar+'.debdelta'

        deltadirname=delta_dirname(new['Filename'],DIR)
        if not os.path.exists(deltadirname): #FIXME this does not respect --no-act
          os.makedirs(deltadirname)
        
        delta=os.path.join(deltadirname,deltabasename)

        free=freespace(deltadirname)
        if free and (free < (newdebsize /2 + 2**15)) :
          if VERBOSE : print ' Not enough disk space for storing ',delta
          continue
        
        if os.path.exists(delta):
          if VERBOSE > 1 : print '  Skip , already exists: ',delta
          continue
        
        if os.path.exists(delta+'-too-big'):
          if VERBOSE > 1 : print '  Skip , tried and too big: ',delta
          continue

        if os.path.exists(delta+'-fails'):
          if VERBOSE > 1 : print '  Skip , tried and fails: ',delta
          continue

        if not ACT:
          print 'Would create:',delta
          continue
        
        if VERBOSE : print 'Creating:',delta
        ret= None
        tdir=tempo()
        try:
          ret=do_delta_(old['File'],new['File'], delta, TD=tdir)
          (delta_, percent, elaps, info_delta, gpg_hashes) = ret
          #insert a first time, it is needed for the testing
          append_info(delta,info_delta)
        except KeyboardInterrupt:
          if os.path.exists(delta):
            os.unlink(delta)
          rmtree(tdir)
          raise
        except DebDeltaError,s:
          if not VERBOSE : print 'Creating:',delta
          print ' Creation of delta failed, reason: ',str(s)
          if os.path.exists(delta):
            os.unlink(delta)
          if not s.retriable :
            open(delta+'-fails','w').close()
          exitstatus=max(exitstatus, s.exitcode)
          ret = None
        except:
          exitstatus=4
          if os.path.exists(delta):
            os.unlink(delta)
          puke( " *** Error while creating delta  "+delta)
          open(delta+'-fails','w').close()
          ret = None

        rmtree(tdir)

        if ret == None:
          continue

        assert(delta == delta_)
        info_delta.append('ServerID: '+HOSTID)
        info_delta.append('ServerBogomips: '+str(BOGOMIPS))
        
        if MAX_DELTA_PERCENT and  percent > MAX_DELTA_PERCENT:
            os.unlink(delta)
            if VERBOSE : print ' Warning, too big!'
            open(delta+'-too-big','w').close()
            continue

        if DO_TEST :
          ##patch test
          pret=None
          try:
            #test, ignoring gpg, that is added later on
            pret=do_patch(delta,old['File'],None , info=info_delta, do_gpg=None)
          except DebDeltaError,s:
            print ' Error: testing of delta failed: ',str(s)
            if os.path.exists(delta):
              os.unlink(delta)
            if not  s.retriable :
              open(delta+'-fails','w').close()
          except KeyboardInterrupt:
            if os.path.exists(delta):
              os.unlink(delta)
            raise
          except Exception,s:
            exitstatus=max(exitstatus,4)
            puke(" *** Error while testing delta  "+delta,s)
            if os.path.exists(delta):
              os.unlink(delta)
            open(delta+'-fails','w').close()
          if pret == None:
            continue
          
          (newdeb_,p_elaps)=pret
          info_delta.append('PatchTime: %.2f' % p_elaps)
          ##end patch test
        #complete the delta file prepending to it the info (a 2nd time)
        try:
          hashes_info=append_info(delta,info_delta)
          # sign ghe delta
          if DO_GPG:
            gpg_hashes['info']=hashes_info
            sign_delta(delta,gpg_hashes)
        except:
          puke('debdeltas')
          if os.path.exists(delta):
            os.unlink(delta)

    #delete debs in --alt that are too old
    if CLEAN_ALT:
      while l>=0:
        old=info_pack[l]
        if old['Label'] == 'ALT':
          f=old['File']
          if os.path.exists(f):
            if VERBOSE : print ' Removing alt deb: ',f
            if ACT: os.unlink(f)
        l-=1

  if VERBOSE : print ' Total running time: %.1f ' % ( -start_time + time.time())

  return exitstatus

##################################################### delta-upgrade

class Predictor:
  package_stats = None
  upgrade_stats = None
  def __init__(self):
    import shelve
    #self.shelve=shelve
    if os.getuid() == 0:
      basedir='/var/lib/debdelta'
    else:
      if not os.path.exists(os.path.expanduser('~/')):
        print '(home directory does not exists, Predictor disabled)'
        return
      basedir=os.path.expanduser('~/.debdelta')

    s=os.path.join(basedir,'upgrade.db')
    if not os.path.exists(basedir):
      print 'Creating:',basedir
      os.makedirs(basedir)
    self.upgrade_stats=shelve.open(s,flag='c')

    s=os.path.join(basedir,'packages_stats.db')
    
    if  os.path.exists(s) or DEBUG > 1 :
      self.package_stats=shelve.open(s,flag='c')

    self.patch_time_predictor=self.patch_time_predictor_math

  ##### predictor for patching time
  def patch_time_predictor_simple(self,p):
    if 'ServerBogomips' in p and 'PatchTime' in p:
      return (float(p[ 'PatchTime']) / BOGOMIPS * float(p['ServerBogomips']) )
    else:
      return None

  def update(self,p,t):
    #save delta info
    if self.package_stats != None :
      n=p['NEW/Package']
      d=copy(p)
      d['LocalDeltaTime']=t
      try:
        self.package_stats[n]=d
      except Exception,exc:
        print 'ERROR:Cannot update package_stats:',exc

    if self.upgrade_stats == None :
      return

    s='ServerID'
    if s not in p :
      return
    s=s+':'+p[s]
    if s not in self.upgrade_stats:
      r=1
      if 'ServerBogomips' in p :
        r=   float(p['ServerBogomips']) / BOGOMIPS
      try:
        self.upgrade_stats[s]={ 'PatchSpeedRatio' : r }
      except Exception,exc:
        print 'ERROR:Cannot update upgrade_stats:',exc

    if 'PatchTime' not in p:
      return
    ut=float(p[ 'PatchTime'])

    r=self.upgrade_stats[s]['PatchSpeedRatio']
    
    nr =  0.95 * r + 0.05 * (  t / ut )
    a=self.upgrade_stats[s]
    a['PatchSpeedRatio'] = nr
    try:
      self.upgrade_stats[s]=a
    except Exception,exc:
      print 'ERROR:Cannot update upgrade_stats:',exc
    if VERBOSE > 1 :
      print '  Upstream ',ut,'PatchSpeedRatio from ',r,' to ',nr
      print self.upgrade_stats[s]['PatchSpeedRatio']
      
  def patch_time_predictor_math(self,p):
    "Predicts time to patch."
    if 'PatchTime' not in p:
      return None
    ut=float(p[ 'PatchTime'])
    #
    s='ServerID'
    if s not in p :
      return self.patch_time_predictor_simple(p)
    s=s+':'+p[s]
    if s not in self.upgrade_stats:
      return self.patch_time_predictor_simple(p)

    r=self.upgrade_stats[s]['PatchSpeedRatio']
    return r * ut

def delta_uri_from_config(config, **dictio):
  secs=config.sections()
  for s in secs:
    opt=config.options(s)
    if 'delta_uri' not in opt:
      raise DebDeltaError('sources.conf section '+repr(s)+'does not contain delta_uri',exitcode=3)
    match=True
    for a in dictio:
      #damn it, ConfigParser changes everything to lowercase !
      if ( a.lower() in opt ) and ( dictio[a] != config.get( s, a) ) :
        #print '!!',a, repr(dictio[a]) , ' != ',repr(config.get( s, a))
        match=False
        break
    if match:
      return  config.get( s, 'delta_uri' )
  if VERBOSE : print ' (sources.conf does not provide a server for ', repr(dictio['PackageName']),')'



def delta_upgrade_(args):
  # a list of all error exitcodes that derive from downloading and applying
  mainexitcodes = [0]
  
  original_cwd = os.getcwd()

  import thread, threading, Queue, pickle, urllib2, fcntl, atexit, signal

  proxies=urllib2.getproxies()
  if VERBOSE and proxies:
    print ' Proxy settings detected in the environment; using "urllib2" for downloading; but'
    print '  this disables some features and is in general slower and buggier. See man page.'
  #for example, urllib2 transforms http response "401"  into "404" , and "302" into "200"
  
  config=ConfigParser.SafeConfigParser()
  a=config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf')  ])
  # FIXME this does not work as documented in Python
  #if VERBOSE > 1 : print 'Read config files: ',repr(a)
  
  import warnings
  warnings.simplefilter("ignore",FutureWarning)
  
  if DO_PROGRESS:
    sys.stderr.write('Initializing APT cache...\r')
  
  try:
    import  apt_pkg
  except ImportError:
    raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt',True)
  
  try:
    import  apt
  except ImportError:
    raise DebDeltaError('python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12).',True)
  
  apt_pkg.init()

  from apt import SizeToStr

  if DO_PROGRESS:
    sys.stderr.write('Upgrading APT cache...\r')

  cache=apt.Cache()
  cache.upgrade(True)

  if DO_PROGRESS:
    sys.stderr.write('Upgraded APT cache.   \r')

  diversions=scan_diversions()

  if DIR == None:
    if os.getuid() == 0:
      DEB_DIR='/var/cache/apt/archives'
    else:
      DEB_DIR='/tmp/archives'
  else:
    DEB_DIR=DIR
  if not os.path.exists(DEB_DIR):
    os.mkdir(DEB_DIR)
  if not os.path.exists(DEB_DIR+'/partial'):
    os.mkdir(DEB_DIR+'/partial')
    
  try:
    ##APT does (according to strace)
    #open("/var/cache/apt/archives/lock", O_RDWR|O_CREAT|O_TRUNC, 0640) = 17
    #fcntl64(17, F_SETFD, FD_CLOEXEC)        = 0
    #fcntl64(17, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=0, len=0}) = 0
    ##so
    a=os.open(DEB_DIR+'/lock', os.O_RDWR | os.O_TRUNC | os.O_CREAT, 0640)
    fcntl.fcntl(a, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
    # synopsis lockf(   fd, operation, [length, [start, [whence]]])
    fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0)
  except IOError, s:
    if s.errno == 11 :
      a=' already locked!'
    else:
      a=str(s)
    if DEB_DIR == '/var/cache/apt/archives' :
      a=a+' (is APT running?)'
    raise DebDeltaError('could not lock dir: '+DEB_DIR+' '+a, retriable=True)
    
  print 'Recreated debs are saved in ',DEB_DIR

  #these are the packages that do not have a delta
  no_delta = []

  total_time = -time.time()

  ##### predictor for patching time
  if DO_PREDICTOR:
    predictor = Predictor()

  #this is a dictonary (key is package name) of parameters of deltas
  #(to add some math in the future)
  params_of_delta={}
  
  patching_queue=Queue.Queue()
  thread_returns={}
  ######################## thread_do_patch
  def thread_do_patch(que, no_delta, returns, exitcodes):
      if VERBOSE > 1 : print '  Patching thread started. '
      debs_size=0
      debs_time=0
      while 1:
        a = que.get()
        if a == None:
          break
        (name, delta , newdeb, deb_uri) = a
        debs_time -= time.time()
        TD=tempo()
        if not ACT:
          print 'Would create: ',newdeb,'   '
        else:
          if VERBOSE >= 2 : print '  Now patching for: ',name
          try:
            start_time=time.time()
            returns['patchname']=os.path.basename(newdeb)
            ret=do_patch_(delta, '/', newdeb , TD, returns,
                         diversions=diversions, do_progress=False)
            del returns['patchname']
            l = os.path.getsize(newdeb)
            a=time.time() - start_time
            if DO_PROGRESS:
              if  terminalcolumns:
                sys.stderr.write(' ' * (terminalcolumns-2) +'\r')
              sys.stderr.write("Created,    time: %.2fsec speed: %4s/sec : %s\n" % \
                  (a , SizeToStr(l / (a+0.001)) , os.path.basename(newdeb)))
            else:
              print  "Created,    time: %.2fsec speed: %4s/sec : %s " % \
                  (a , SizeToStr(l / (a+0.001)) , os.path.basename(newdeb))
          except KeyboardInterrupt:
            thread.interrupt_main()
            rmtree(TD)
            return
          except DebDeltaError,s:
            print ' Error: applying of delta for ',name,'failed: ',str(s)
            if 'e' in DEB_POLICY:
              no_delta.append( (deb_uri, newdeb) )
            elif VERBOSE : print ' No deb-policy "e", no download of ',deb_uri
            exitcodes.append(s.exitcode)
          except:
            puke( " *** Error while applying delta for "+name+": ")
            if 'e' in DEB_POLICY:
              no_delta.append( (deb_uri, newdeb) )
            elif VERBOSE : print ' No deb-policy "e", no download of ',deb_uri
            exitcodes.append(4)
          else:
            if name in params_of_delta :
              p= params_of_delta[name]
              name,elaps=ret
              if DO_PREDICTOR:
                predictor.update(p,elaps)
                if VERBOSE > 1 :
                  t=predictor.patch_time_predictor(p)
                  if t: print '  (Predicted %.3f sec )'  % t
            debs_size += os.path.getsize(newdeb)
            if os.path.exists(delta):
              os.unlink(delta)
        rmtree(TD)
        debs_time += time.time()
      returns['debs_size']=debs_size
      returns['debs_time']=debs_time
      if VERBOSE > 1 : print '  Patching thread ended , bye bye. '
      
  #####################################
  
  def progress_string(statusdb):
    download=''
    if 'downloaduri' in statusdb:
      download="D %2d%% (%4s/s) %s " % \
          (statusdb.get('downloadprogress',-1),
           statusdb.get('downloadspeed','-'),
           statusdb['downloaduri'])
    patch=''
    if 'patchname' in statusdb:
      patch='P %2d%% %s'% (statusdb.get('patchprogress',-1), statusdb['patchname'])
    if terminalcolumns == None:
      return download+' ; '+patch
    if not patch:
      return download[:(terminalcolumns-1)]
    if not download:
      return patch[:(terminalcolumns-1)]
    ld=len(download)
    lp=len(patch)
    b=ld + lp
    if b < terminalcolumns - 4 :
      return download+' ; '+patch
    a=float(terminalcolumns-4)/float(b)
    ld=int(ld*a)
    lp=int(lp*a)
    return download[:ld] + ' ; ' + patch[:lp]
  
  #########################################

  import socket, httplib
  from urlparse import urlparse

  #################### manage connections
  #keeps a cache of all connections, by URL
  http_conns={}
  
  def conn_by_url(url):
    url=urlparse(url)[1]
    if url not in http_conns:
      if VERBOSE > 1 : print '  Opening connection to: ',url
      http_conns[url] = httplib.HTTPConnection(url)
    return http_conns[url]
  
  def conn_close(url,fatal=False):
    url=urlparse(url)[1]
    conn=http_conns.get(url)
    if fatal:
      http_conns[url] = None
    else:
      del http_conns[url]
    if conn != None :
      if VERBOSE > 1 : print '  Closing connection to: ',url
      conn.close()

  ####

  def _connect(uri, headers):
    "connects for a GET ; returns (filetype, statuscode, servermessage, getheaders)"
    uri_p=urlparse(uri)
    if uri[:7] == 'http://' and not proxies:
      #use persistent http connections
      conn=conn_by_url(uri)
      if conn == None :
        return None, None, None, None
      try:
        conn.request("GET", urllib2.quote(uri_p[2]),headers=headers)
        r = conn.getresponse()
        return r, r.status, r.reason, r.msg
      except (httplib.HTTPException, socket.error),e:
        if VERBOSE : puke( ' Connection error (retrying): ',uri_p[1])
        conn_close(uri)
        try:
          conn=conn_by_url(uri)
          conn.request("GET", urllib2.quote(uri_p[2]),headers=headers)
          r = conn.getresponse()
          return r, r.status, r.reason, r.msg
        except (httplib.HTTPException, socket.error),e:
          puke( 'Connection error (fatal): ',uri_p[1])
          mainexitcodes.append(1)
          try:
            conn_close(uri,fatal=True)
          except: pass
          mainexitcodes.append(1)
          return e, None, None, None
    else: #use urllib2
      try:
        req = urllib2.Request(uri, headers=headers)
        r = urllib2.urlopen(req)
        #print r.info(),dir(r),r.code
        return r, getattr(r,'code',None), getattr(r,'msg','(no message)'), r.info()
      except urllib2.HTTPError,e:
        return e.code, None, None, None
      except (httplib.HTTPException,socket.error,urllib2.URLError),e:
        puke( 'Connection error (fatal)',uri)
        mainexitcodes.append(1)
        return e, None, None, None

  ################################################# various HTTP facilities

  def _parse_ContentRange(s):
    #bytes 0-1023/25328
    if not s or s[:6] != "bytes " :
      print "Malformed Content-Range",s
      return
    a=s[6:].split('/')
    if len(a) != 2 :
      print "Malformed Content-Range",s
      return
    b=a[0].split('-')
    if len(b) != 2 :
      print "Malformed Content-Range",s
      return
    return int(b[0]),int(b[1]),int(a[1])
  ###################################### test_uri
  def test_uri(uri):
      conn=conn_by_url(uri)
      if conn == None: return None
      uri_p=urlparse(uri)
      assert(uri_p[0] == 'http')
      conn.request("HEAD", urllib2.quote(uri_p[2]),headers=HTTP_USER_AGENT)
      r = conn.getresponse()
      r.read()
      r.close()
      return r.status

  ###################################### download_1k_uri
  def download_1k_uri(uri,outname):
      "in case of connection error, returns the (error, None, None) ; otherwise returns (status,len,outname)"
      #download
      uri_p=urlparse(uri)
      assert(uri_p[0] == 'http')
      re=copy(HTTP_USER_AGENT)
      re["Range"] =  "bytes=0-1023"
      r, status, msg, responseheaders=_connect(uri, re)
      if not hasattr(r,'read') and responseheaders==None:
        return r, None, None
      if status == 206:
        outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname))
        try:
          l = _parse_ContentRange(responseheaders['Content-Range'])[2]
        except (KeyError, ValueError):
          l = None
      elif status == 200:
        outnametemp=outname
        try:
          l=long(responseheaders.get('Content-Length'))
        except:
          l=None
      else: #FIXME how do we deal with a FTP mirror of deltas ?
        r.read()
        r.close()
        return status, None, None
      if os.path.exists(outnametemp) and os.path.getsize(outnametemp) >= 1023 :
        #do not truncate preexisting file
        r.read()
        r.close()
        return status, outnametemp, l
      out=open(outnametemp,'w')
      out.write(r.read())
      out.close()
      r.close()
      return status, outnametemp, l

  ###################################### download_uri
  def download_uri(uri, outname, conn_time, len_downloaded, statusdb):
      outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname))
      re=copy(HTTP_USER_AGENT)
      #content range
      l=None
      if os.path.exists(outnametemp):
        #shamelessly adapted from APT, methods/http.cc
        s=os.stat(outnametemp)
        l=s[ST_SIZE]
        #t=s[ST_MTIME]
        ### unfortunately these do not yet work
        #thank god for http://docs.python.org/lib/module-time.html
        #actually APT does
        #t=time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t))
        ##re["If-Range"] =  time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t))
        ####re["If-Range"] =  time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(t))
        re["Range"] =  "bytes=%li-" % ( (long(l)-1) )
      #start downloading
      start_time=time.time()
      r, status, message, responseheaders = _connect(uri, re)
      if not hasattr(r,'read') and responseheaders==None:
        return 
      if not ( status == None or status == 200 or ( status == 206 and l != None ) ):
        if VERBOSE :
          print 'Connection problem, status:'+str(status)+' msg:'+str(message)+' uri:'+str(uri)
        r.read()
        r.close()
        return
      #print 'ooK Content-Range', r.getheader('Content-Range') #HACK
      if l and status == 200 :
        print ' Hmmm... our HTTP range request failed, ',repr(re),status,message
      if status == 200 :
        out=open(outnametemp,'w')
        try:
          total_len = long(responseheaders['Content-Length'])
        except (KeyError,ValueError):
          total_len=None
      elif status == 206 :
        #APT does scanf of    "bytes %lu-%*u/%lu",&StartPos,&Size
        #first-byte-pos "-" last-byte-pos "/" instance-length
        out=open(outnametemp,'a')
        try:
          a,b,total_len =_parse_ContentRange(responseheaders['Content-Range'])
        except (KeyError,ValueError),e:
          sys.stderr.write('! problem, http response [206], Content Range %s , error %s , uri %s\n' %\
                           (responseheaders.get('Content-Range'),e,uri))
          return
        out.seek(a)
        out.truncate()
      else:
        out=open(outnametemp,'w')
        try:
          total_len=long(responseheaders.get('Content-length'))
        except ValueError:
          total_len=None
      
      free=freespace(os.path.dirname(outname))
      if total_len != None and free and (free + 2**14 ) < total_len  :
        print 'Not enough disk space to download: ',os.path.basename(uri)
        r.read()
        r.close()
        mainexitcodes.append(1)
        return
      j=0
      
      s=r.read(1024)
      while s and (total_len == None or out.tell() < total_len ):
        j+=len(s)
        out.write(s)
        if total_len:
          statusdb['downloadprogress']=99.9 * out.tell() / total_len
        a=time.time() + conn_time - start_time
        if a  > 0.5 :
          statusdb['downloadspeed']=SizeToStr(float(j+len_downloaded)/a)
        s=r.read(1024)
      out.close()
      r.close()
      #end of download
      a=time.time() - start_time
      #if total_len == None:
      #  total_len = os.path.getsize(outnametemp)
      if not DO_PROGRESS:
        print "Downloaded, time: %.2fsec speed: %4s/sec : %s " % \
              (a , SizeToStr(j / (a+0.001)) , os.path.basename(uri))
      else:
        if terminalcolumns:
          sys.stderr.write(' ' * (terminalcolumns-2) +'\r')
        sys.stderr.write("Downloaded, time: %.2fsec speed: %4s/sec : %s\n" % \
                         (a , SizeToStr(j / (a+0.001)) , os.path.basename(uri)))
      
      os.rename(outnametemp,outname)
      #FIXME this is incorrect by 1024 bytes
      return  conn_time + a, (j+len_downloaded)

  ###################################### end of HTTP stuff
  
  deltas_down_size=0
  deltas_down_time=0

  #this is a list of tuples of .....
  available_deltas=[]

  not_available_deltas=[]

  #distinguish python-apt version 0.7.7 from 0.7.10
  newer_python_apt = hasattr(apt.package,'Version')
  
  progress_count=0
  
  ## first merry-go-round, use package cache to fill available_deltas
  for p in cache :
    #print progress
    if DO_PROGRESS :
      progress_count+=1
      if 0 == (progress_count & 63):
        sys.stderr.write('%2.1f%% %s\r' % ((float(progress_count) * 100.0 / len(cache)), ' '*max(1,terminalcolumns-7)))

    if p.isInstalled and p.markedUpgrade :
      if args and p.name not in args:
        continue
      if newer_python_apt:
        #thanks a lot to Julian Andres Klode
        candidate=p.candidate
        origin = p.candidate.origins[0]
        arch=candidate.architecture
        deb_uri=candidate.uri
        installed_version=p.installed.version
        candidate_version=p.candidate.version
        deb_path=string.split(deb_uri,'/')
        try:
          thepoolindex=deb_path.index('pool')
        except ValueError:
          sys.stderr.write('! Package "%s" (version %s) does not have "pool" in the uri %s \n' % (p.name, candidate_version, deb_uri))
          continue
        deb_path=string.join(deb_path[(thepoolindex):],'/')
      else:
        #thanks a lot to Michael Vogt
        p._lookupRecord(True)
        dpkg_params = apt_pkg.ParseSection(p._records.Record)
        arch = dpkg_params['Architecture']
        origin = p.candidateOrigin[0]
        candidate_version = p.candidateVersion
        installed_version = p.installedVersion
        deb_path = dpkg_params['Filename']
        for (packagefile,i) in p._depcache.GetCandidateVer(p._pkg).FileList:
          indexfile = cache._list.FindIndex(packagefile)
          if indexfile:
            deb_uri=indexfile.ArchiveURI(deb_path)
            break
      
      #try all possible variants of the filename
      newdebs=[p.name+'_'+candidate_version+'_'+arch+'.deb',
               os.path.basename(deb_uri)]
      if ':' in candidate_version:
        a=candidate_version.split(':')
        newdebs.append(p.name+'_'+a[1]+'_'+arch+'.deb')
        newdebs.append(p.name+'_'+a[0]+'%3A'+a[1]+'_'+arch+'.deb')
        newdebs.append(p.name+'_'+a[0]+'%3a'+a[1]+'_'+arch+'.deb')
      
      for newdeb in newdebs:
        if os.path.exists(DEB_DIR+'/'+newdeb) or \
            os.path.exists('/var/cache/apt/archives/'+newdeb):
          if VERBOSE > 1 : print  '  Already downloaded: ',p.name,candidate_version
          newdeb=None
          break
      if newdeb == None:
        continue
      newdeb = DEB_DIR+'/'+newdebs[-1]
      
      if VERBOSE > 1 : print '  Looking for a delta for %s from %s to %s ' % ( p.name, installed_version, candidate_version )
      delta_uri_base=delta_uri_from_config(config,
                                           Origin=origin.origin,
                                           Label=origin.label,
                                           Site=origin.site,
                                           Archive=origin.archive,
                                           PackageName=p.name)
      if delta_uri_base == None:
        if 's' in DEB_POLICY:
          no_delta.append( (deb_uri, newdeb) )
        continue

      a=urlparse(delta_uri_base)
      assert(a[0] == 'http')

      #delta name
      delta_name=p.name+'_'+version_mangle(installed_version)+\
                  '_'+ version_mangle(candidate_version)+'_'+\
                  arch+'.debdelta'

      uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name
      
      #download first part of delta
      abs_delta_name= DEB_DIR+'/'+delta_name

      #maybe it is already here
      if os.path.exists(abs_delta_name):
        a = abs_delta_name
      else:
        a = DEB_DIR+'/partial/'+delta_name
        if not os.path.exists(a):
          a = None
      if a:
        l=os.path.getsize(a)
        if VERBOSE > 1 : print '  Already here: ',abs_delta_name
        s=get_info_fast(a)
        if s:
          params_of_delta[p.name]=info_2_db(s)
        available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, a )  )
        continue
      #if not, download its first part

      if DO_PROGRESS:
        sys.stderr.write('%2.1f%% Downloading head of %s...   \r' % \
                           ((float(progress_count) * 100.0 / len(cache)),
                         p.name))
      deltas_down_time-=time.time()
      status, tempname, l = download_1k_uri(uri,abs_delta_name)
      deltas_down_time+=time.time()

      #some strange error in remote server?
      #FIXME this does not support ftp delta repositories
      if status != 200 and status != 206 and status != 404: 
        print 'Debdelta is not downloadable (%s %s):%s' %\
              (status,httplib.responses.get(status), uri)
        continue

      if status == 404: 
        not_available_deltas.append(p.name)
        if uri[:7] == 'http://' and not proxies: # FIXME support ftp or proxies
          bigrstatus = test_uri(uri+'-too-big')
        else:
          bigrstatus = None
        if bigrstatus == 200 :
          print 'Debdelta is too big:', delta_name
          if 'b' in DEB_POLICY:
            no_delta.append( (deb_uri, newdeb) )
          elif VERBOSE : print ' No deb-policy "b", no download of ',deb_uri
        else:
          print 'Debdelta is not present:', delta_name
          if 'u' in DEB_POLICY:
            no_delta.append( (deb_uri, newdeb) )
          elif VERBOSE : print ' No deb-policy "u", no download of ',deb_uri
        continue

      if VERBOSE:
          print 'Debdelta is present:', delta_name, tempname,'    '
      elif DO_PROGRESS:
        sys.stderr.write('%2.1f%% Downloaded head of %s.      \r' % \
                           ((float(progress_count) * 100.0 / len(cache)),
                         p.name))

      if os.path.isfile(tempname):
        deltas_down_size+=os.path.getsize(tempname)
      
      #parse file and save info
      try:
        s=get_info_fast(tempname)
      except DebDeltaError,e:
        sys.stderr.write("!!"+str(e)+'\n')
        sys.stderr.write("!! (renamed to "+tempname+'~~NOT~A~DELTA~~  )\n')
        os.rename(tempname,tempname+'~~NOT~A~DELTA~~')
        if proxies:
          sys.stderr.write("!!maybe a proxy is returning an error page??\n")
        else:
          sys.stderr.write("!!damaged delta??\n")
        continue
      if s:
        params_of_delta[p.name]=info_2_db(s)
        s=patch_check_tmp_space(params_of_delta[p.name],  '/')
        if s != True:
          print p.name,' : sorry '+s
          #neither download deb nor delta..
          #the user may wish to free space and retry
          continue
      #FIXME may check that parameters are conformant to what we expect

      available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, tempname  ) )
  ## end of first merry-go-round

  available_deltas.sort()

  if not available_deltas:
    print 'No deltas available to be downloaded/applied.'
    return


  if DEBUG or VERBOSE:
    print ' So far, downloaded, time: %.2fsec size: %s speed: %4s/sec' % \
        (deltas_down_time, SizeToStr(deltas_down_size), SizeToStr(deltas_down_size/float(deltas_down_time+0.001)))

  if DEBUG or VERBOSE:
    print 'Deltas: %d present and %d not.' % (len(available_deltas), len(not_available_deltas))
    print 'Need to get %s of deltas.' %  SizeToStr(sum([a[0] for a in available_deltas]))

  ############# start downloading and progress thread

  patching_thread=threading.Thread(
    target=thread_do_patch  ,
    args=(patching_queue, no_delta, thread_returns, mainexitcodes) )
  patching_thread.start()
  
  def print_progress(common_db):
    while 'STOP' not in common_db:
      time.sleep(0.2)
      sys.stderr.write(progress_string(common_db)+'\r')
  
  if DO_PROGRESS and terminalcolumns > 4:
    progress_thread=threading.Thread(target=print_progress, args=( thread_returns, ) )
    progress_thread.start()
  else:
    progress_thread=None
  
  ## second merry-go-round, try downloading available delta
  for delta_len, name, uri, abs_delta_name , newdeb, deb_uri, tempname  in available_deltas :
    if  not os.path.exists(abs_delta_name) and os.path.exists(tempname) and os.path.getsize(tempname) == delta_len:
      print 'just Rename ',name
      os.rename(tempname,abs_delta_name)

    if name in params_of_delta:
      s=patch_check_tmp_space(params_of_delta[name],  '/')
      if s != True:
        print name,' : sorry, '+s
        #argh, we ran out of space in meantime
        continue
    
    if not os.path.exists(abs_delta_name):
      thread_returns['downloaduri']=os.path.basename(uri)
      r=download_uri(uri, abs_delta_name, deltas_down_time, deltas_down_size, thread_returns)
      del thread_returns['downloaduri']
      if r == None or isinstance(r, httplib.HTTPException) :
        if VERBOSE : print ' You may wish to rerun,  to get also: ',uri
        continue
      else:
        deltas_down_time = r[0]
        deltas_down_size = r[1]

      #queue to apply delta
    if os.path.exists(abs_delta_name):
        #append to queue
        patching_queue.put( (name, abs_delta_name  ,newdeb, deb_uri ) )
    else:
      if 'u' in DEB_POLICY:
        no_delta.append( (deb_uri, newdeb) )
      elif VERBOSE : print ' No deb-policy "u", no download of ',deb_uri
  ## end of second merry-go-round

  #terminate queue
  patching_queue.put(None)
  
  #do something useful in the meantime
  debs_down_size=0
  debs_down_time=0
  if patching_thread.isAlive() and no_delta and VERBOSE > 1 :
    print '  Downloading deltas done, downloading debs while waiting for patching thread.'
  while patching_thread.isAlive() or ('a' in DEB_POLICY and no_delta):
    if no_delta:
      uri, newdeb  = no_delta.pop()
      thread_returns['downloaduri']=os.path.basename(uri)
      r=download_uri(uri, newdeb, debs_down_time, debs_down_size, thread_returns)
      del thread_returns['downloaduri']
      if r == None or isinstance(r, httplib.HTTPException) :
        if VERBOSE : print ' You may wish to rerun, to get also: ',uri
        continue
      if r:
        debs_down_time = r[0]
        debs_down_size = r[1]
    if not no_delta : 
      time.sleep(0.1)
  
  for i in http_conns:
    if http_conns[i] != None :
      http_conns[i].close()
  
  while patching_thread.isAlive():
    time.sleep(0.1)
  
  #terminate progress report
  thread_returns['STOP']=True
  while progress_thread != None and progress_thread.isAlive():
    time.sleep(0.1)
  
  total_time += time.time()
  print 'Delta-upgrade statistics:'
  if VERBOSE:
    if deltas_down_time :
      a=float(deltas_down_size)
      t=deltas_down_time
      print ' download deltas size %s time %dsec speed %s/sec' %\
            ( SizeToStr(a) , int(t), SizeToStr(a / t ))
    if thread_returns['debs_time'] :
      a=float(thread_returns['debs_size'])
      t=thread_returns['debs_time']
      print ' patching to debs size %s time %dsec speed %s/sec' %\
            ( SizeToStr(a) , int(t), SizeToStr(a / t ))
    if debs_down_time :
      a=float(debs_down_size)
      t=debs_down_time
      print ' download debs size %s time %dsec speed %s/sec' %\
            ( SizeToStr(a) , int(t), SizeToStr(a / t ))
  if total_time:
    a=float(debs_down_size  + thread_returns['debs_size'])
    print ' total resulting debs size %s time %dsec virtual speed: %s/sec' %  \
          ( SizeToStr(a ), int(total_time), SizeToStr(a / total_time))
    
  return max(mainexitcodes)

################################################# main program, do stuff

def act():
  "fake function that marks where the action starts"
  pass

if action == 'patch':
  if INFO  :
    if  len(argv) > 1 and VERBOSE :
      sys.stderr.write(' (printing info - extra arguments are ignored)\n')
    elif  len(argv) == 0  :
      sys.stderr.write('Need a  filename ;  try --help\n')
      raise SystemExit(3)
    try:
        delta=abspath(argv[0])
        check_is_delta(delta)
        info=get_info(delta)
        for s in info:
          if s:
            print ' info: ',s
    except KeyboardInterrupt:
        puke('debpatch exited by keyboard interrupt')
        raise SystemExit(5)
    except DebDeltaError,s:
        puke('debpatch',s)
        raise SystemExit(s.exitcode)
    except Exception,s:
        puke("debpatch",s)
        raise SystemExit(4)
    raise SystemExit(0)
  #really patch
  if len(argv) != 3 :
    sys.stderr.write('Need 3 filenames ;  try --help\n')
    raise SystemExit(3)

  newdeb=abspath(argv[2])
  if newdeb == '/dev/null':
      newdeb = None

  try:
    do_patch(abspath(argv[0]), abspath(argv[1]), newdeb)
  except KeyboardInterrupt:
    puke('debpatch exited by keyboard interrupt')
    raise SystemExit(5)
  except DebDeltaError,s:
    puke('debpatch',s)
    raise SystemExit(s.exitcode)
  except Exception,s:
    puke('debpatch',s)
    raise SystemExit(4)
  raise SystemExit(0)

elif action == 'delta' :
  if len(argv) != 3 :  
    sys.stderr.write('Need 3 filenames ;  try --help\n')
    raise SystemExit(3)
  
  delta=abspath(argv[2])
  try:
    r = do_delta(abspath(argv[0]), abspath(argv[1]), delta)
  except KeyboardInterrupt:
    puke('debdelta exited by keyboard interrupt')
    raise SystemExit(5)
  except DebDeltaError,s:
    puke('debdelta',s)
    raise SystemExit(s.exitcode)
  except Exception, s:
    puke('debdelta',s)
    raise SystemExit(4)
  raise SystemExit(0)

elif action == 'deltas' :
  try:
    exitcode=do_deltas(argv)
  except KeyboardInterrupt:
    puke('debdeltas exited by keyboard interrupt')
    raise SystemExit(5)
  except DebDeltaError,s:
    puke('debdeltas',s)
    raise SystemExit(s.exitcode)
  except Exception,s:
    puke('debdeltas',s)
    raise SystemExit(4)
  raise SystemExit(exitcode)

elif action == 'delta-upgrade':
  import warnings
  warnings.simplefilter("ignore",FutureWarning)
  try:
    exitcode=delta_upgrade_(argv)
  except KeyboardInterrupt:
    puke('debdelta-upgrade exited due to keyboard interrupt')
    raise SystemExit(5)
  except DebDeltaError,s:
    puke('debdelta-upgrade',s)
    raise SystemExit(s.exitcode)
  except Exception, s:
    puke('debdelta-upgrade',s)
    raise SystemExit(4)
  raise SystemExit(exitcode)

elif action == 'patch-url':
  config=ConfigParser.SafeConfigParser()
  config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf')  ])

  try:
    import  apt_pkg
  except ImportError:
    print 'ERROR!!! python module "apt_pkg" is missing. Please install python-apt'
    raise SystemExit(1)
  
  try:
    import  apt
  except ImportError:
    print 'ERROR!!! python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12)'
    raise SystemExit(1)
  
  apt_pkg.init()
  
  cache=apt.Cache()
  cache.upgrade(True)

  for a in argv:
    print 'Lookup ',a
    p = cache[a]
    candidate=p.candidate
    origin = p.candidate.origins[0]
    arch=candidate.architecture
    if not candidate.uris :
      print 'Sorry, cannot find an URI to download the debian package of ',a
      continue
    deb_uri = candidate.uri
    installed_version=p.installed.version
    candidate_version=p.candidate.version
    deb_path=string.split(deb_uri,'/')
    deb_path=string.join(deb_path[(deb_path.index('pool')):],'/')

    delta_uri_base=delta_uri_from_config(config,
                                         Origin=origin.origin,
                                           Label=origin.label,
                                           Site=origin.site,
                                           Archive=origin.archive,
                                           PackageName=p.name)

    if delta_uri_base == None:
      print 'Sorry, no debdelta source is available to upgrade ',a
      continue

    if installed_version == candidate_version:
      print 'Sorry, this package is already at its newest version ',a
      continue

    #delta name
    delta_name=p.name+'_'+version_mangle(installed_version)+\
                '_'+ version_mangle(candidate_version)+'_'+\
                arch+'.debdelta'
  
    uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name

    print 'The package ',a,' may be upgraded by using: ', uri
  raise SystemExit(0)

##################################################### apt method

### still work in progress
if  os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
  import os,sys, select, fcntl, apt, thread, threading, time

  apt_cache=apt.Cache()
  
  log=open('/tmp/log','a')
  log.write('  --- here we go\n')
  
  ( hi, ho , he) = os.popen3('/usr/lib/apt/methods/http.distrib','b',2)

  nthreads=3

  class cheat_apt_gen:
    def __init__(self):
      self.uri=None
      self.filename=None
      self.acquire=False
    def process(self,cmd):
      if self.uri:
        self.filename=cmd[10:-1]
        log.write(' download %s for %s\n' % (repr(self.uri),repr(self.filename)))
        self.uri=None
        self.filename=None
        self.acquire=False
        return cmd
      elif self.acquire:
        self.uri=cmd[5:-1]
        return cmd
      elif cmd[:3] == '600' :
        self.acquire=True
      else:
        return cmd
  
  def copyin():
    bufin=''
    while 1:
      #print ' o'
      s=os.read(ho.fileno(),1)
      bufin += s
      if log and bufin and (s == '' or s == '\n') :
        log.write( ' meth ' +repr(bufin)+'\n' )
        bufin=''
      if s == '':
        thread.interrupt_main(   )
        global nthreads
        if nthreads:
          nthreads-=1
        #log.write( ' in closed \n' )
        #return
      os.write(1,s)


  def copyerr():
    buferr=''
    while 1:
      s=os.read(he.fileno(),1)
      buferr += s
      if log and buferr and (s == '' or s == '\n') :
        log.write( ' err ' +repr(buferr)+'\n' )
        buferr=''
      if s == '':
        thread.interrupt_main(   )
        global nthreads
        if nthreads:
          nthreads-=1
        log.write( ' err closed \n' )
        #return
      os.write(2,s)

  def copyout():
    gen=cheat_apt_gen()
    bufout=''
    while 1:
      s=os.read(0,1)
      bufout += s
      if log and bufout and (s == '' or s == '\n') :
        log.write( ' apt ' +repr(bufout)+'\n' )

        bufout=gen.process(bufout) 
        
        bufout=''
      if s == '':
        thread.interrupt_main()
        global nthreads
        if nthreads:
          nthreads-=1
        #log.write( ' out closed \n' )
        #return
      os.write(hi.fileno(),(s))

        
  tin=thread.start_new_thread(copyin,())
  tout=thread.start_new_thread(copyout,())
  terr=thread.start_new_thread(copyerr,())
  while nthreads>0 :
    log.write( ' nthreads %d \n' % nthreads )
    try:
      while nthreads>0 :
        time.sleep(1)      
    except KeyboardInterrupt:
      pass
  raise SystemExit(0)

@


1.244
log
@stat_to_tar : import pwd grp (!!)
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.243 2009/12/07 16:00:11 debdev Exp $"
d1360 1
a1360 1
            ret=os.system('/usr/sbin/prelink -u '+a)
d1362 1
a1362 1
            ret=os.system('/usr/sbin/prelink -u '+a+' >/dev/null 2>&1')
d1365 2
@


1.243
log
@debdelta_upgrade : do not redownload partial downloads of deltas
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.242 2009/12/07 15:51:16 debdev Exp $"
d1001 1
d1009 2
a1010 1
    else:    
@


1.242
log
@debdelta_upgrade : progress report in first round of head downloading
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.241 2009/12/06 22:05:32 debdev Exp $"
d3684 1
d3687 7
a3693 1
        l=os.path.getsize(abs_delta_name)
d3695 1
a3695 1
        s=get_info_fast(abs_delta_name)
d3698 1
a3698 1
        available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, abs_delta_name )  )
@


1.241
log
@do_patch : other adjustments to forensic
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.240 2009/12/06 21:46:18 debdev Exp $"
d3184 3
d3201 3
d3207 3
d3248 1
a3248 3
  start_sec = time.time()
  len_deltas=0

d3593 2
d3597 3
a3599 1

d3602 6
d3695 4
d3711 1
d3728 7
d3765 4
d3774 4
d3865 1
a3865 1
  elaps =  time.time() - start_sec
d3883 1
a3883 1
  if elaps:
d3886 1
a3886 1
          ( SizeToStr(a ), int(elaps), SizeToStr(a / elaps))
@


1.240
log
@do_patch : really write forensic
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.239 2009/12/06 19:02:26 debdev Exp $"
d1462 2
a1463 2
  temp_err_name_fd, temp_err_name = tempfile.mkstemp(prefix='debdelta')
  temp_name_fd, temp_name = tempfile.mkstemp(prefix='debdelta')
d1517 1
a1517 2
                       #delta+'\n!! '+
                       temp_fore_name+'\n!! '+temp_name+'\n!! '+temp_err_name+'\n')
@


1.239
log
@mv removal of old file from delta_files__ to delta_files
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.238 2009/11/15 17:38:38 debdev Exp $"
d1465 1
a1465 1
                     stderr=subprocess.PIPE)
d1469 1
d1486 6
d1498 2
a1499 1
                       delta+'\n!! '+ olddeb+'\n!! '+temp_name+'\n!! '+temp_err_name+'\n')
d1519 1
@


1.238
log
@patch: correct bug when progress reporting but no debname
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.237 2009/11/15 17:33:03 debdev Exp debdev $"
d1817 1
d1822 1
a1822 1
      script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+' ; rm '+p+'\n')    
d1827 1
a1827 1
      script.write('~/debdelta/zdelta-2.1/zdu '+o+' '+p+' '+n+' ; rm '+p+'\n')
d1832 1
a1832 1
      script.write('~/debdelta/bdelta-0.1.0/bpatch '+o+' '+n+' '+p+' ; rm '+p+'\n')
d1837 1
a1837 1
      script.write('~/debdelta/diffball-0.7.2/patcher '+o+' '+p+' '+n+' ; rm '+p+'\n')
d1842 1
a1842 1
      script.write('rdiff patch '+o+' '+p+' '+n+' ; rm '+p+'\n')
d1846 1
a1846 1
      script.write('xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
d1853 1
a1853 1
      script.write('bspatch '+o+' '+n+' '+p+'; rm '+p+'\n')
d1859 1
a1859 1
      script.write('bunzip2 '+p+'.bz2 ; xdelta patch '+p+' '+o+' '+n+' ; rm '+p+'\n')
d1863 1
a1863 1
      script.write('xdelta patch '+p+' '+o+' '+n+' ; rm '+p+'\n')
d1866 1
a1866 1
      script.write('~/debdelta/jdiff06/src/jpatch '+o+' '+p+' '+n+' ; rm '+p+'\n')
d1896 1
a1896 1
    script.write('rm '+o+'\n')
@


1.237
log
@unzip, script_zip : piped version
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.236 2009/11/08 20:18:54 debdev Exp $"
d1171 1
d1173 1
d1341 1
a1341 1
        sys.stderr.write('P %2d%% %s\r' % (progress, os.path.basename(newdeb)))
d1473 1
a1473 1
      sys.stderr.write('P %2d%% %s\r' % (progress, os.path.basename(newdeb)))
@


1.236
log
@HTTPResponse instance has  no attribute 'message'
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.235 2009/10/29 07:12:46 debdev Exp $"
d1773 1
a1773 1
  def unzip(f, in_script_as_well = None):
a1776 2
      if in_script_as_well or ( in_script_as_well == None and f[:3] != 'NEW' ):
        script.write('gunzip '+f+'\n')
a1780 2
      if in_script_as_well or ( in_script_as_well == None and f[:3] != 'NEW' ):
        script.write('bzip2 '+f+'\n')
a1785 2
      if in_script_as_well or ( in_script_as_well == None and f[:3] != 'NEW' ):
        script.write('lzma '+f+'\n')
d1788 1
a1788 1
    else: assert(0)
d1791 9
a1799 2
  def script_zip(n,cn,newhead=None):
    "inverts the unzip() function ; optionally, forces .gz header (to fight changes in libz)"
d1803 1
a1803 1
        script.write("$E '"+ s +"' >> "+n+cn +' && ./minigzip -9 < '+n+' | tail -c +'+str(len(newhead)+1)+' >> '+n+cn+' && rm '+n+' \n')
d1805 1
a1805 1
        script.write('./minigzip -9 < '+n+' >> '+n+cn+' && rm '+n+'\n')
d1808 1
a1808 1
      script.write('./minibzip2 -9 < '+n+' >> '+n+cn+' && rm '+n+'\n')
d1811 1
a1811 1
      script.write('lzma -9 < '+n+' >> '+n+cn+' && rm '+n+'\n')
d2157 1
a2157 1
      (old_filename,old_filename_ext) = unzip(old_filename,False)
@


1.235
log
@patch : eliminate useless wrong line to compute progress
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.234 2009/10/28 20:15:06 debdev Exp $"
d3383 1
a3383 1
          return r, r.status, r.message, r.msg
@


1.234
log
@patch :  fore() : delete unused argument
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.233 2009/10/28 20:13:43 debdev Exp $"
d1336 1
a1336 1
      progress=(int(6.0 + 6.0 * float(progressline) / progresslen))
a1339 1
      runtime['patchprogress']=float(progress)/float(len(s))*10.0
@


1.233
log
@patch : delete useless 'if'
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.232 2009/10/28 20:00:33 debdev Exp $"
d1485 1
a1485 1
    def fore(ret=None):
d1488 1
a1488 1
    def fore(ret=None):
d1492 1
a1492 1
    def fore(ret=None):
d1519 1
a1519 1
      fore(ret)
@


1.232
log
@patch : correct logic in generating forensic when '-d' and localepurge
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.231 2009/10/28 19:11:46 debdev Exp $"
d1486 1
a1486 2
      if ret != 2:
        sys.stderr.write('(Faulty delta. Please consider retrying with the option "-d" ).\n')
d1489 2
a1490 3
      if ret != 2:
        sys.stderr.write('!!Faulty delta. Please send by email to '+EMAIL+' the files:\n!! '+
                         delta+'\n!! '+ olddeb+'\n!! '+temp_name+'\n!! '+temp_err_name+'\n')
d1507 3
a1509 4
      if ret != 2:
        sys.stderr.write('!!Faulty delta. Please send by email to '+EMAIL+' the files:\n!! '+
                         #delta+'\n!! '+
                         temp_fore_name+'\n!! '+temp_name+'\n!! '+temp_err_name+'\n')
@


1.231
log
@patch : introduce some intermediate progress numbers
 (though usually they run away quite fast)
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.230 2009/10/28 18:31:28 debdev Exp $"
d1509 1
a1509 1
      if ret != 2 and not localepurged and not HAVE_LOCALEPURGE:
a1516 1
    fore(ret)
d1522 1
@


1.230
log
@patch : report progress of symlink tree creation
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.229 2009/10/28 18:25:36 debdev Exp $"
d1193 3
a1195 1
  
d1209 3
a1211 1
  
d1225 2
d1252 2
d1278 2
d1336 1
a1336 1
      progress=(int(10.0 * float(progressline) / progresslen))
d1398 3
d1455 2
d1469 1
a1469 1
    progress=(int(10.0 + 89.9 * progresschar / progresslen))
d1481 1
d1546 2
@


1.229
log
@do_patch : reset patch progress at start
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.228 2009/10/28 18:24:56 debdev Exp $"
d1317 1
a1317 1
  def _symlink_data_tree(pa,TD,diversions):
d1322 2
d1325 6
d1395 1
a1395 1
        linked_file_pairs,diverted=_symlink_data_tree(params['OLD/Package'],TD,diversions)
d1454 1
a1454 1
    progress=(int(99.9 * progresschar / progresslen))
@


1.228
log
@def parse_prelink_conf(): just a draft
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.227 2009/10/28 18:23:13 debdev Exp $"
d1180 1
@


1.227
log
@patch :
  call prelink only on e_type in ('ET_DYN','ET_EXEC')  (speeds
    up kernel patching quite a lot)
  and if prelink is ctrl-c interrupted, exit
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.226 2009/10/25 13:58:54 debdev Exp $"
d1124 22
@


1.226
log
@debdelta-upgrade : print "Created..." string similar to download
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.225 2009/10/25 13:47:46 debdev Exp $"
d1086 37
d1306 6
a1311 1
        if HAVE_PRELINK and '\x7fELF' == open(divert).read(4):
d1315 2
a1316 1
            os.system('/usr/sbin/prelink -u '+a)
d1318 3
a1320 1
            os.system('/usr/sbin/prelink -u '+a+' >/dev/null 2>&1')
@


1.225
log
@download_uri() : correct final speed computation in case of partial downloads
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.224 2009/10/25 13:39:35 debdev Exp $"
d3168 1
d3173 4
a3176 2
            if VERBOSE == 0 :
              if DO_PROGRESS and  terminalcolumns:
d3178 5
a3182 1
              print 'Created:', os.path.basename(newdeb),'   '
@


1.224
log
@change proxy warning to refer to man page
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.223 2009/10/25 12:37:31 debdev Exp $"
d3454 2
a3455 2
      if total_len == None:
        total_len = os.path.getsize(outnametemp)
d3458 1
a3458 1
              (a , SizeToStr(total_len / (a+0.001)) , os.path.basename(uri))
d3463 1
a3463 1
                         (a , SizeToStr(total_len / (a+0.001)) , os.path.basename(uri)))
@


1.223
log
@wait for progress thread and patching thread to exit at the end
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.222 2009/10/25 11:29:54 debdev Exp $"
d3071 1
a3071 2
    print '  this disables the downloading of debs when the delta is too big,'
    print '  (see "--deb-policy" in man page) and is in general slower and buggier.'
@


1.222
log
@report download progress also in extra download step '--deb-policy a'
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.221 2009/10/25 11:27:34 debdev Exp $"
d3641 1
a3641 1
    while True:
d3648 2
a3686 2
  if patching_thread.isAlive():
    time.sleep(0.2)
d3712 8
@


1.221
log
@use "urllib2" for http with proxies, and for any other URL (such as FTP)
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.220 2009/10/24 15:03:01 debdev Exp $"
d3696 1
d3698 1
@


1.220
log
@implement progress reporting in debdelta-upgrade
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.219 2009/10/24 13:07:43 debdev Exp $"
d312 1
a312 1
    freespace= a[0] * a[4]
d3066 1
a3066 1
  import thread, threading, Queue, pickle, urllib, fcntl, atexit, signal
d3068 7
a3247 5
  def __host_by_url__(url):
    if url[:7] == 'http://' :
      url = urlparse(url)[1]
    return url

d3249 1
a3249 1
    url=__host_by_url__(url)
d3256 1
a3256 1
    url=__host_by_url__(url)
d3266 43
a3308 1
  
a3309 13
  def _http_whine_(uri,r):
    a='Url'
    if uri[-9:] == '.debdelta':
      a='Debdelta'
    elif not VERBOSE: #do not report non-existence of other URIs
      return
    if  r.status == 200 or  r.status == 206:
      pass
    if r.status == 404:
      print a,' is not present:',uri
    else:
      print a,' is not available (',repr(r.status), r.reason,'):', uri

d3311 1
a3311 1
  def _parse_ContentRange(r):
d3313 1
a3313 3
    s=r.getheader('Content-Range')
    if s == None: return
    if s[:6] != "bytes " :
d3328 1
d3331 1
a3331 1
      conn.request("HEAD", urllib.quote(uri_p[2]),headers=HTTP_USER_AGENT)
d3335 1
a3335 1
      return r
d3339 1
a3339 1
      "in case of connection error, returns the error ; otherwise returns (request,outname)"
d3345 4
a3348 12
      try:
        conn=conn_by_url(uri)
        if conn == None : return
        conn.request("GET", urllib.quote(uri_p[2]),headers=re)
        r = conn.getresponse()
      except (httplib.HTTPException, socket.error),e:
        puke('Connection error: ',e)
        conn_close(uri)
        mainexitcodes.append(1)
        return e
      #print '1K Content-Range', r.getheader('Content-Range') #HACK
      if r.status == 206:
d3350 5
a3354 1
      elif r.status == 200:
d3356 5
a3360 1
      else:
d3363 1
a3363 1
        return r, False
d3368 1
a3368 1
        return r, outnametemp
d3373 1
a3373 1
      return r, outnametemp
a3376 2
      uri_p=urlparse(uri)
      assert(uri_p[0] == 'http')
d3395 6
a3400 22
      try:
        conn=conn_by_url(uri)
        if conn == None : return
        conn.request("GET", urllib.quote(uri_p[2]),headers=re)
        r = conn.getresponse()
      except (httplib.HTTPException,socket.error),e:
        if VERBOSE : puke( ' Connection error (retrying): ',e)
        try:
          conn_close(uri)
          conn=conn_by_url(uri)
          if conn == None : return
          conn.request("GET", urllib.quote(uri_p[2]),headers=re)
          r = conn.getresponse()
        except (httplib.HTTPException,socket.error),e:
          puke( 'Connection error (fatal): ',e)
          mainexitcodes.append(1)
          try:
            conn_close(uri,fatal=True)
          except: pass
          return e
      if not ( r.status == 200 or ( r.status == 206 and l != None ) ):
        if VERBOSE : _http_whine_(uri,r)
d3403 1
a3403 1
        return None
d3405 3
a3407 11
      if l and r.status == 200 :
        print ' Hmmm... our HTTP range request failed, ',repr(re),r.status,r.reason
      assert( r.length == int(r.getheader('content-length')) )
      free=freespace(os.path.dirname(outname))
      if free and (free + 2**14 ) < r.length  :
        print 'Not enough disk space to download: ',os.path.basename(uri)
        r.read()
        r.close()
        mainexitcodes.append(1)
        return None
      if r.status == 200 :
d3409 5
a3413 2
        total_len = r.length
      elif r.status == 206 :
a3415 1
        a,b,total_len =_parse_ContentRange(r)
d3417 6
d3425 14
d3440 1
d3442 1
a3442 1
      while s and out.tell() < total_len :
d3445 2
a3446 1
        statusdb['downloadprogress']=99.9 * out.tell() / total_len
d3455 2
d3458 2
a3459 1
        print "Downloaded, time: %.2fsec speed: %4s/sec uri: %s " % (a , SizeToStr(total_len / (a+0.001)) , uri)
d3463 3
a3465 1
        sys.stderr.write("Downloaded:  %s \n" % os.path.basename(uri) )
d3571 1
a3571 1
      r = download_1k_uri(uri,abs_delta_name)
a3573 6
      if  r == None or isinstance(r, httplib.HTTPException) or isinstance(r, socket.error) :
        if VERBOSE : print ' You may wish to rerun, to get also: ',uri
        continue
      
      r,tempname = r

d3575 4
a3578 2
      if r.status != 200 and r.status != 206 and r.status != 404: 
        print 'Debdelta is not downloadable (',repr(r.status), repr(r.reason),'): ', uri
d3581 6
a3586 3
      if r.status == 404:
        bigr = test_uri(uri+'-too-big')
        if bigr.status == 200 :
a3600 5
      #get remote file length
      if r.status == 206:
        l = _parse_ContentRange(r) [2]
      else:
        l=int(r.getheader('content-length'))
d3602 11
a3612 1
      s=get_info_fast(tempname)
d3697 1
a3697 1
      if isinstance(r, httplib.HTTPException) :
@


1.219
log
@implement progress report in do_patch_()
@
text
@d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.218 2009/10/24 12:58:26 debdev Exp $"
d1090 1
d1093 2
a1094 1
    r=do_patch_(delta,olddeb,newdeb,TD=T, info=info, diversions=diversions, do_gpg=do_gpg)
d1103 2
a1104 1
def do_patch_(delta,olddeb,newdeb, TD, info=None, diversions=None, do_gpg=DO_GPG):
d1379 2
a1380 2
    #runtime['patchprogress']=progress
    if DO_PROGRESS:
d1383 2
d3156 1
d3162 8
a3169 2
            ret=do_patch(delta,'/',newdeb , diversions=diversions)
            if VERBOSE == 0 : print 'Created:',os.path.basename(newdeb),'   '
d3172 1
d3198 1
d3203 30
d3346 1
a3346 1
  def download_uri(uri,outname,conn_time,len_downloaded):
d3365 2
d3413 1
a3413 3
      a=time.time()
      conn_time-=a
      j=out.tell()
d3415 1
a3415 1
      while s and j < total_len :
d3418 4
a3421 6
        if not DEBUG and a + 0.5 < time.time() :
          a=time.time()
          sys.stderr.write("%d%% (%4s/s) %s \r" % \
                           (100*j / total_len,
                            SizeToStr((j+len_downloaded)/(a+conn_time)),\
                            os.path.basename(uri)[:50] ))
d3425 3
a3427 3
      conn_time+=time.time()
      if DEBUG:
        a = time.time() - a
d3430 2
d3435 1
a3435 1
      return  conn_time , (j+len_downloaded)      
d3536 2
d3539 1
d3565 3
d3591 7
d3603 9
d3626 3
a3628 1
      r=download_uri(uri , abs_delta_name , deltas_down_time,deltas_down_size)
d3659 1
a3659 1
      r=download_uri(uri , newdeb, debs_down_time, debs_down_size )
@


1.218
log
@use module 'threading'
@
text
@d108 12
d148 1
d170 1
a170 1
RCS_VERSION="$Id: debdelta,v 1.217 2009/10/24 12:43:20 debdev Exp $"
d1366 18
a1383 2
  temp_name, temp_err_name, ret=  system(SHELL+' -e '+a+' PATCH/patch.sh', TD,
                                         ignore_output=True, return_output=True)
@


1.217
log
@use module 'queue'
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.216 2009/10/18 14:08:10 debdev Exp $"
d3032 1
a3032 1
  import  thread , Queue, pickle, urllib, fcntl, atexit, signal
d3112 1
a3112 1
  def thread_do_patch(que, threads, no_delta, returns, exitcodes):
a3159 1
      threads.pop()
d3512 4
a3515 3
  threads=[]
  threads.append(thread.start_new_thread(thread_do_patch  ,
                 (patching_queue, threads, no_delta, thread_returns, mainexitcodes) ) )
d3551 1
a3551 1
  if threads:
d3557 1
a3557 1
  if  threads and no_delta and VERBOSE > 1 :
d3559 1
a3559 1
  while threads or ('a' in DEB_POLICY and no_delta):
@


1.216
log
@do not crash if package is found in a local repository , w/o
"pool". Thanks Nelson A. de Oliveira for spotting.
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.215 2009/09/22 17:35:30 debdev Exp $"
d3032 1
a3032 1
  import  thread , pickle, urllib, fcntl, atexit, signal
d3109 1
a3109 1
  (qout,qin)=os.pipe()
d3112 1
a3112 1
  def thread_do_patch(qout,threads,no_delta,returns,exitcodes):
d3117 4
a3120 7
        s=os.read(qout,1)
        c=''
        while s != '\t' :
          c+=s
          s=os.read(qout,1)
        if c == '\t' or c == '': break
        (name, delta , newdeb, deb_uri) = pickle.loads(c)
d3515 1
a3515 1
                 (qout,threads,no_delta, thread_returns, mainexitcodes) ) )
d3542 1
a3542 2
        c=pickle.dumps(  (name, abs_delta_name  ,newdeb, deb_uri ) )
        os.write(qin, c + '\t' )
d3550 1
a3550 1
  os.write(qin,'\t\t\t')
@


1.215
log
@debdelta-upgrade :
  reviewed the functions test_uri ()   download_1k_uri() ;
  if the delta is absent, there is one 'print' that tells why
  (and if it is too big)
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.214 2009/09/22 17:21:06 debdev Exp $"
d3393 6
a3398 1
        deb_path=string.join(deb_path[(deb_path.index('pool')):],'/')
@


1.214
log
@avoid extra spaces in printout
add column in 'Created:'
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.213 2009/09/18 13:39:58 debdev Exp $"
a3233 1
      _http_whine_(uri,r)
d3236 1
a3236 3
      if r.status == 200:
        return r
      return False
d3240 2
a3261 1
        _http_whine_(uri,r)
d3264 1
a3264 1
        return False
d3266 1
a3271 1
      #print '1K OK', outnametemp, out.tell() #HACK
d3452 1
d3461 1
d3468 11
a3478 3
      if not r:
        r = test_uri(uri+'-too-big')
        if r :
d3482 5
a3486 3
        elif 'u' in DEB_POLICY:
          no_delta.append( (deb_uri, newdeb) )
        elif VERBOSE : print ' No deb-policy "u", no download of ',deb_uri
d3489 1
a3489 2
      r,tempname = r
      
d3491 1
a3491 1
        a,b,l = _parse_ContentRange(r)
d3494 1
@


1.213
log
@add checks against localepurge, avoid useless forensic
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.212 2009/09/12 14:29:18 debdev Exp $"
d2811 1
a2811 1
        if VERBOSE : print 'Creating :',delta
d2825 1
a2825 1
          if not VERBOSE : print 'Creating: ',delta
d2929 1
a2929 1
      print 'Creating: ',basedir
d3131 1
a3131 1
            if VERBOSE == 0 : print 'Created ',newdeb,'   '
d3206 1
a3206 1
      print a,' is not present: ',uri
d3208 1
a3208 1
      print a,' is not available (',repr(r.status), r.reason,'): ', uri
@


1.212
log
@patch.sh : start with #!/bin/bash , delete snippet for choosing echo
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.211 2009/09/11 12:37:07 debdev Exp $"
d1093 2
a1094 1

d1240 1
d1246 1
a1246 1
    for orig,divert in s:          
d1263 7
a1269 2
      else:
        if VERBOSE > 3 : print '    not symlinking ',divert,' to ',orig
d1346 4
d1359 3
a1361 2
    def fore():
      sys.stderr.write('(Faulty delta. Please consider retrying with the option "-d" ).\n')
d1363 4
a1366 3
    def fore():
      sys.stderr.write('!!Faulty delta. Please send by email to '+EMAIL+' the files:\n!! '+
                       delta+'\n!! '+ olddeb+'\n!! '+temp_name+'\n!! '+temp_err_name+'\n')
d1368 1
a1368 1
    def fore():
d1375 3
d1381 6
a1386 5
      except Exception,s:
        sys.stderr.write('!!While creating forensic '+temp_name+' error:'+str(s)+'\n')
      sys.stderr.write('!!Faulty delta. Please send by email to '+EMAIL+' the files:\n!! '+
                       #delta+'\n!! '+
                       temp_fore_name+'\n!! '+temp_name+'\n!! '+temp_err_name+'\n')
d1391 7
a1397 2
    fore()
    raise DebDeltaError('error in patch.sh.')
@


1.211
log
@delta_gzipped_files() : use shorter name for temporary
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.210 2009/09/11 12:35:08 debdev Exp $"
d398 2
a399 2
echo='echo -ne'
if test c`$echo 'i'"$c" `o = ciiao  ; then
d402 2
a403 2
 echo='echo -n'
 if test c`$echo  'i'"$c" `o = ciiao  ; then 
d407 2
a408 2
  echo='/bin/echo -ne'
  test c`$echo  'i'"$c" `o = ciiao  
d433 1
a433 1
    a=ECHO_TEST  + " $echo '" + repres +  "' \n exit "
d1476 1
a1476 1
  script.write('#!/bin/sh -e\n')
d1656 1
a1656 1
        script.write("$echo  '"+ s +"' >> "+n+cn +' && ./minigzip -9 < '+n+' | tail -c +'+str(len(newhead)+1)+' >> '+n+cn+' && rm '+n+' \n')
d1973 1
a1973 1
  script.write(ECHO_TEST)
d2233 1
a2233 1
      script.write("$echo '"+ s +"'\"${FTH}\" >> OLD/mega_cat\n")
d2287 1
a2287 1
    script.write("$echo '"+ s +"' >> NEW.file\n")
@


1.210
log
@delta_gzipped_files() : use pipes and 'gzip -c' to avoid overwriting another stupid copy of the same file (see bug 546172)
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.209 2009/09/11 11:46:11 debdev Exp $"
d1843 1
a1843 1
    p='PATCH/tmp_gzip'
@


1.209
log
@skip empty line in list of conffiles
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.208 2009/09/07 15:25:10 debdev Exp $"
d1872 1
a1872 1
    script.write("mv "+p+".new '"+o[:-3]+"' ;  gzip "+gzip_flags+" '"+o[:-3]+"'\n")
@


1.208
log
@delta_tar() : rewrite skipping rules
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.207 2009/09/07 15:19:04 debdev Exp $"
d1557 1
a1557 1
    old_conffiles=[ de_bar(a) for a in p.read().split('\n') ]
@


1.207
log
@matches sections in debdelta.conf to the package name, not filename
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.206 2009/09/07 15:16:50 debdev Exp $"
d2019 17
a2035 1
      oldname = oldtarinfo.name
d2039 1
a2039 1
          if VERBOSE > 2 : print '  skipping ',repr(oldname),' as per rule ',repr(j)
d2042 1
a2042 7
        
      if  (oldname in skip) or shell_not_allowed(oldname) or \
             not oldtarinfo.isreg() or oldtarinfo.size == 0:
        continue
      if VERBOSE > 3 and oldname != de_bar(oldname):
        print '     filename in old tar has weird ./ in front: ' , oldname 
      oldname = de_bar(oldname)
d2044 1
d2046 1
@


1.206
log
@align VERBOSE and add a missing print
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.205 2009/09/05 15:08:17 debdev Exp $"
a1442 11
  debdelta_conf=ConfigParser.SafeConfigParser()
  debdelta_conf.read(['/etc/debdelta/debdelta.conf', expanduser('~/.debdelta/debdelta.conf')  ])

  debdelta_conf_skip=[]
  for s in debdelta_conf.sections():
    if fnmatch.fnmatch(os.path.basename(olddeb),s):
      opt=debdelta_conf.options(s)
      if 'skip' in opt:
        debdelta_conf_skip += debdelta_conf.get(s,'skip').split(';') 
      break
  
d1501 12
@


1.205
log
@forensic : do not ask for the complete delta, but save its name and sha1
also, do print delta filepath on error, and lowercase start
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.204 2009/09/04 22:01:19 debdev Exp $"
d780 1
a780 2
    if VERBOSE > 1 :
      print '  (Warning, cannot get info from old style: '+delta+' )'
d1124 1
a1124 2
      if VERBOSE:
        print ' The signature by "'+a[4:]+'" is correctly verified for ',delta
d1505 1
a1505 2
      if  VERBOSE  :
        sys.stdout.write(' '+o+': '+join([o[4:] for o in  s],' ')+'\n')
d1750 1
a1750 1
            ( ( deltasize * 100. /  nsize ) , n, (nsize / 1024. / ( tim + 0.001 )))
d1841 1
a1841 1
      if VERBOSE: print ' Warning: unknown OS in .gz format: ',oct(ord(a[9])),n
d1862 1
a1862 1
      if VERBOSE > 2: print '   warning: cannot re-gzip to equal file: ',r,n
d1868 1
a1868 2
    if VERBOSE > 2 :
      print '   ',n[9:],'  (= to %d%%): ' % (100*before/newsize) ,
d2022 1
a2022 1
          if VERBOSE > 2 : '  skipping ',repr(oldname),' as per rule ',repr(j)
d2457 1
a2457 1
    if VERBOSE: print ' Using avoid dict ',AVOID
d2521 1
a2521 1
          if VERBOSE > 2: print '   seen twice, do not re-add info: ',f
d2715 1
a2715 2
    if VERBOSE>2:
      print '   Package: ',pa,' Versions:',versions
d2723 1
a2723 2
        if VERBOSE > 1 :
          print '  Newest version deb was not in cmdline, skip down one: ', new['File']
d2734 1
a2734 1
      if VERBOSE > 1:     print '  Skip , too small: ', new['File']
d2761 1
a2761 1
          if VERBOSE > 1:     print '  Skip , already exists: ',delta
d2765 1
a2765 1
          if VERBOSE > 1:     print '  Skip , tried and too big: ',delta
d2769 1
a2769 1
          if VERBOSE > 1:     print '  Skip , tried and fails: ',delta
d2776 1
a2776 1
        if VERBOSE: print 'Creating :',delta
d2790 1
a2790 1
          if not VERBOSE: print 'Creating: ',delta
d2868 1
a2868 1
            if VERBOSE: print ' Removing alt deb: ',f
d2872 1
a2872 1
  if VERBOSE: print ' Total running time: %.1f ' % ( -start_time + time.time())
d2987 1
a2987 2
  if VERBOSE:
    print ' (sources.conf does not provide a server for ', repr(dictio['PackageName']),')'
d3093 1
a3093 1
          if VERBOSE>=2 : print '  Now patching for: ',name
d3104 1
a3104 1
            elif VERBOSE: print ' No deb-policy "e", no download of ',deb_uri
d3110 1
a3110 1
            elif VERBOSE: print ' No deb-policy "e", no download of ',deb_uri
d3127 1
a3127 2
      if VERBOSE > 1 :
        print '  Patching thread ended , bye bye. '
d3145 1
a3145 2
      if VERBOSE > 1:
        print '  Opening connection to: ',url
d3157 1
a3157 2
      if VERBOSE > 1 :
        print '  Closing connection to: ',url
d3395 1
a3395 3
      if VERBOSE > 1:
        print '  Looking for a delta for %s from %s to %s ' % \
              ( p.name, installed_version, candidate_version )
d3438 1
a3438 1
          elif VERBOSE: print ' No deb-policy "b", no download of ',deb_uri
d3441 1
a3441 1
        elif VERBOSE: print ' No deb-policy "u", no download of ',deb_uri
d3500 1
a3500 1
      elif VERBOSE: print ' No deb-policy "u", no download of ',deb_uri
@


1.204
log
@disable predictor (it was never really used)
@
text
@d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.203 2009/09/04 21:56:50 debdev Exp $"
d1362 2
a1363 2
        #temp_file.write('Delta: '+delta+'\n')
        #temp_file.write('DeltaSHA1: '+hash_to_hex(sha1_hash_file(delta))+'\n')
d1370 2
a1371 1
                       delta+'\n!! '+temp_fore_name+'\n!! '+temp_name+'\n!! '+temp_err_name+'\n')
d1377 1
a1377 1
    raise DebDeltaError(' Error in applying patch.sh in: '+delta)
d1384 1
a1384 1
      raise DebDeltaError('New deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'])
@


1.203
log
@better printouts for forensic '-d'
@
text
@d151 2
d157 1
a157 1
RCS_VERSION="$Id: debdelta,v 1.202 2009/09/04 18:00:43 debdev Exp $"
d3073 2
a3074 1
  predictor = Predictor()
d3122 5
a3126 4
              predictor.update(p,elaps)
              if VERBOSE > 1 :
                t=predictor.patch_time_predictor(p)
                if t: print '  (Predicted %.3f sec )'  % t
a3535 2

  #save predictor...
@


1.202
log
@add forensic for -d
@
text
@d6 2
d155 1
a155 1
RCS_VERSION="$Id: debdelta,v 1.201 2009/09/04 15:51:11 debdev Exp $"
d520 1
d537 3
d541 1
a541 1
  os.close(temp_err_fd)
d543 1
a543 5
    if return_output:
      return temp_name
    else:
      os.unlink(temp_name)
      return
d1053 3
d1109 6
a1114 3
  o=system('ar xvo '+delta,  TD+'/PATCH', return_output=True, ignore_output=True)
  ar_list_delta=[a[4:] for a in open(o).read().split('\n') if a]
  os.unlink(o)
d1339 7
d1347 4
a1350 3
  if olddeb != '/':
    def fore(): pass
  elif DEBUG == 0:
d1352 2
a1353 1
      sys.stderr.write('(Faulty delta. Please consider the option "-d" ).\n')
d1356 1
a1356 1
      temp_name=''
d1358 1
a1358 1
        (temp_fd,temp_name) = tempfile.mkstemp(prefix="debforensic_")
d1360 2
a1361 1
        temp_file.write('Delta: '+delta+'\n')
d1364 1
d1366 3
a1368 3
        sys.stderr.write('While creating forensic '+temp_name+' error:'+str(s)+'\n')
      else:
        sys.stderr.write('Faulty delta. Please send by email to mennucc1@@debian.org the forensic file: '+temp_name+'\n')
d1371 2
a1372 6
  a=''
  if VERBOSE > 2 : a = '-v'
  script_time = - time.time()
  try:
    system(SHELL+' -e '+a+' PATCH/patch.sh', TD)
  except:
d1374 1
a1374 2
    raise
  script_time += time.time()
d1394 3
@


1.201
log
@bug: exitstatus is exitcode
@
text
@d96 1
a96 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time, traceback, ConfigParser, subprocess, time
d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.200 2009/09/04 15:18:00 debdev Exp $"
d861 1
d873 195
d1093 4
a1096 1
    
d1136 8
d1188 1
a1188 1
    " 'diversions' must be prepared by scan_diversions() "
d1190 1
d1198 5
a1202 2
        if p != pa:    s.append((a,b))
        else:     s.append((a,a))
d1205 1
a1205 1
    return s
d1208 2
d1211 1
d1224 1
d1229 1
a1229 1
    return s
d1233 1
a1233 1
      s=dpkg_L_faster(pa,diversions)
d1235 1
a1235 1
      s=dpkg_L(pa)
d1255 1
d1277 3
d1288 1
a1288 2
        pa=params['OLD/Package']
        _symlink_data_tree(pa,TD,diversions)
d1308 1
a1308 1
            if os.path.exists(a ):
d1310 1
d1330 21
d1355 5
a1359 1
  system(SHELL+' -e '+a+' PATCH/patch.sh', TD)
d1366 1
d1372 6
a1377 2
      system('echo "'+params['NEW/MD5sum']+'  NEW.file" | md5sum -c', TD,
             ignore_output=True)
d1538 1
@


1.200
log
@increase vebosity for 'bash -v' to -vvv
@
text
@d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.199 2009/09/04 15:15:52 debdev Exp $"
d2532 1
a2532 1
          exitstatus=max(exitstatus, s.exitstatus)
@


1.199
log
@the error for missing xdelta xdelta3 bsdiff lzma is retriable
@
text
@d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.198 2009/09/04 13:42:59 debdev Exp $"
d1113 1
a1113 1
  if VERBOSE > 1 : a = '-v'
@


1.198
log
@debdeltas : --test
@
text
@d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.197 2009/09/04 13:28:11 debdev Exp $"
d1095 1
a1095 1
        die('This patch needs xdelta3. Please install the Debian package "xdelta3".')
d1098 1
a1098 1
        die('This patch needs xdelta. Please install the Debian package "xdelta".')
d1101 1
a1101 1
        die('This patch needs bsdiff. Please install the Debian package "bsdiff".')
@


1.197
log
@indent VERBOSE output according to verbosity;
always use VERBOSE and not DEBUG to control when to print;
rearrange DEBUG meaning;
use stderr for cmdline errors
@
text
@d135 3
d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.196 2009/09/03 21:40:17 debdev Exp $"
d178 1
a178 1
                    'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=") )
d207 1
d2557 1
a2557 1
        if DEBUG :
@


1.196
log
@debdelta-upgrade : in deb filename, quote ':' as '%3a' in versions with epochs
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.195 2009/08/29 06:25:09 debdev Exp $"
d259 1
a259 1
    if VERBOSE > 4: print ' would unlink ',a
d261 1
a261 1
    if VERBOSE > 4: print ' would rmdir ',a
d263 1
a263 1
    if VERBOSE > 4: print ' would rm -r ',a
d273 1
a273 1
      if DEBUG > 2 : raise
d292 2
a293 2
  except:
    if VERBOSE : print ' statvfs error ',a
d453 1
a453 1
    if DEBUG > 1 :
d517 1
a517 1
    print 'Warning "system()" in ',TD,' for ',a
d520 1
a520 1
  if DEBUG > 2 : print '    system(',a,')=',
d522 2
a523 2
  if DEBUG > 2 : print ret
  if DEBUG and ignore_output==False and (os.stat(temp_name)[ST_SIZE] > 0 or os.stat(temp_err_name)[ST_SIZE] > 0 ):
d635 1
a635 1
      print '   GPG> ',j,
d773 1
a773 1
      print '(Warning, cannot get info from old style: '+delta+' )'
d950 1
a950 1
  if  DEBUG or olddeb != '/':
d1026 2
a1027 2
          if VERBOSE > 3 or DEBUG > 1:
            print '   copying/unprelinking ',divert,' to ',a
d1032 1
a1032 1
          if VERBOSE > 3 : print '   symlinking ',divert,' to ',a
d1043 1
a1043 1
      if VERBOSE > 1 : print ' Performing chmod ',n,oct(om),oct(nm)
d1109 1
a1109 1
  if VERBOSE > 3 : a = '-v'
d1235 1
a1235 1
        sys.stdout.write(o+': '+join([o[4:] for o in  s],' ')+'\n')
d1242 1
a1242 1
  if DEBUG > 1 : print '  debdelta.conf says we will skip: ', repr(debdelta_conf_skip)  
d1341 1
a1341 1
    if VERBOSE > 1 :
d1461 1
a1461 1
      print '  Warning, memory usage by bsdiff on the order of %dMb' % (12 * osize / 2**20)
d1478 1
a1478 1
      print '   delta is %3.2f%% of %s, speed: %dkB /sec'  % \
d1752 1
a1752 1
          if DEBUG + VERBOSE > 1 : '  skipping ',repr(oldname),' as per rule ',repr(j)
d1760 1
a1760 1
        print ' Filename in old tar has weird ./ in front: ' , oldname 
d1790 1
a1790 1
        print ' Weird permission: ',newname,oct(a),repr(newtarinfo.type)
d1795 1
a1795 1
        print ' Filename in new tar has weird ./ in front: ' , newname 
d1816 1
a1816 1
          if VERBOSE and o not in skip: print '  Hmmm... there is a md5 but not a file: ',o
d2036 1
a2036 1
    if VERBOSE > 3: print '  ar line: ',repr(s)
d2131 1
a2131 1
  if VERBOSE > 1 : print ' '+patch_files[0][1]+' wins on patch.sh'
d2201 1
a2201 1
      if VERBOSE > 1 :     print 'Avoid: ', f
d2238 1
a2238 1
          if VERBOSE > 2 : print '  skip udeb: ',f
d2251 1
a2251 1
          if DEBUG > 1: print 'Seen twice, do not re-add info: ',f
d2300 1
a2300 1
          if VERBOSE > 2 : print 'Promoting to CMDLINE ',f
d2325 1
a2325 1
      if DEBUG > 1 : print ' no such delta dir ',f
d2423 1
a2423 1
  if VERBOSE > 1 : print ' total parsing time: %.1f ' % ( -start_time + time.time())
d2441 1
a2441 1
            if VERBOSE: print 'Removing: ',f_d          
d2446 1
a2446 1
      print 'Package: ',pa,' Versions:',versions
d2455 1
a2455 1
          print 'Newest version deb was not in cmdline, skip down one: ', new['File']
d2489 1
a2489 1
          if VERBOSE : print 'Not enough disk space for storing ',delta
d2600 1
a2600 1
            if VERBOSE: print 'Removing alt deb: ',f
d2604 1
a2604 1
  if VERBOSE: print 'Total running time: %.1f ' % ( -start_time + time.time())
d2620 1
a2620 2
        if DEBUG:
          print ' Home directory does not exists, Predictor disabled.'
d2632 1
a2632 1
    if  os.path.exists(s) or DEBUG :
d2685 1
a2685 1
      print ' Upstream ',ut,'PatchSpeedRatio from ',r,' to ',nr
d2720 1
a2720 1
    print '(sources.conf does not provide a server for ', repr(dictio['PackageName']),')'
d2810 1
a2810 1
      if VERBOSE > 1 : print ' Patching thread started. '
d2825 1
a2825 1
          if VERBOSE>=2 : print ' Now patching for: ',name
d2851 1
a2851 1
                if t: print '   (Predicted %.3f sec )'  % t
d2859 1
a2859 1
        print ' Patching thread ended , bye bye. '
d2877 2
a2878 2
      if (DEBUG or VERBOSE > 1) :
        print '-Opening connection to: ',url
d2890 2
a2891 2
      if (DEBUG or VERBOSE > 1)  :
        print '-Closing connection to: ',url
d3003 1
a3003 1
        if DEBUG or VERBOSE > 1 : puke( 'Connection error (retrying): ',e)
d3122 1
a3122 1
          if VERBOSE > 1 : print  'Already downloaded: ',p.name,candidate_version
d3130 1
a3130 1
        print 'Looking for a delta for %s from %s to %s ' % \
d3157 1
a3157 1
        if VERBOSE > 1 : print 'Already here: ',abs_delta_name
d3248 1
a3248 1
    print ' Downloading deltas done, downloading debs while waiting for patching thread.'
d3302 1
a3302 1
      print '(printing info - extra arguments are ignored)'
d3304 1
a3304 1
      print ' need a  filename ;  try --help'
d3325 1
a3325 1
    print ' need 3 filenames ;  try --help'
d3347 1
a3347 1
    print ' need 3 filenames ;  try --help'
@


1.195
log
@use os.path.expanduser on --dir and --gpg-home
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.194 2009/08/28 21:27:52 debdev Exp $"
d3111 16
a3126 4
      newdeb=os.path.basename(deb_uri)
      if os.path.exists(DEB_DIR+'/'+newdeb) or \
             os.path.exists('/var/cache/apt/archives/'+newdeb):
        if VERBOSE > 1 : print  'Already downloaded: ',newdeb
d3128 2
a3129 2
      newdeb = DEB_DIR+'/'+newdeb

@


1.194
log
@check existence of GPG_HOME, and use abspath
@
text
@d99 1
a99 1
from os.path import abspath
d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.193 2009/08/28 21:13:49 debdev Exp $"
d211 1
a211 1
      DIR = abspath(v)
d233 1
a233 1
      GPG_HOME=abspath(v)
d752 1
a752 1
  delta=abspath(delta)
d1173 1
a1173 1
  debdelta_conf.read(['/etc/debdelta/debdelta.conf', os.path.expanduser('~/.debdelta/debdelta.conf')  ])
d2734 1
a2734 1
  a=config.read(['/etc/debdelta/sources.conf', os.path.expanduser('~/.debdelta/sources.conf')  ])
d3385 1
a3385 1
  config.read(['/etc/debdelta/sources.conf', os.path.expanduser('~/.debdelta/sources.conf')  ])
@


1.193
log
@debdeltas: some corrections in exception handling when testing the
 applying of the delta
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.192 2009/08/28 21:05:19 debdev Exp $"
d233 4
a236 1
      GPG_HOME=v
@


1.192
log
@debdelta_upgrade() : return exitcode
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.191 2009/08/28 20:53:42 debdev Exp $"
d2558 2
a2561 2
              if os.path.exists(delta):
                os.unlink(delta)
d2563 2
d2566 3
a2568 3
          except:
            exitstatus=max(exitstatus,2)
            puke(" *** Error while testing delta  "+delta)
d2571 1
a2571 1
          
@


1.191
log
@do_deltas() : try: except around gpg signing
sign_delta() : correct bug
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.190 2009/08/28 20:48:00 debdev Exp $"
d2721 3
d2805 1
a2805 1
  def thread_do_patch(qout,threads,no_delta,returns):
d2833 1
d2839 1
d2950 1
d3008 1
d3027 1
d3187 2
a3188 1
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads,no_delta, thread_returns) ) )
d3274 2
d3366 1
a3366 1
    delta_upgrade_(argv)
d3376 1
a3376 1
  raise SystemExit(0)
d3386 1
a3386 1
    raise SystemExit
d3392 1
a3392 1
    raise SystemExit
@


1.190
log
@puke() : do not print None traces
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.189 2009/08/28 20:45:55 debdev Exp $"
d711 2
a712 1
    r=os.system("ar qS "+delta+" "+TD+"/_gpg"+role)
d718 1
a718 1
    raise DebDeltaError('GnuPG fails to sign, returncode '+str(p.returncode))
d2551 1
d2575 1
d2577 10
a2586 5
        hashes_info=append_info(delta,info_delta)
        # sign ghe delta
        if DO_GPG:
          gpg_hashes['info']=hashes_info
          sign_delta(delta,gpg_hashes)
@


1.189
log
@do_deltas() : report exitstatus
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.188 2009/08/28 20:14:47 debdev Exp $"
d575 1
a575 1
  if DEBUG :
@


1.188
log
@cannot raise None
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.187 2009/08/28 20:11:49 debdev Exp $"
d2156 2
d2524 1
d2527 1
d2563 1
d2593 2
d3331 1
a3331 1
    do_deltas(argv)
d3341 1
a3341 1
  raise SystemExit(0)
@


1.187
log
@puke on stderr
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.186 2009/08/28 20:08:30 debdev Exp $"
d1441 1
a1441 1
    else: raise
@


1.186
log
@protect code in sign_delta() with a try: except
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.185 2009/08/28 20:00:41 debdev Exp $"
d572 1
d574 3
a576 3
  if VERBOSE or e == '':    print s,' : ',e,str(typ),str(value)
  else: print s,' : ',e
  if DEBUG : print traceback.print_tb(trace)
@


1.185
log
@do_deltas : change the order of the code, to avoid signing twice with gpg
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.184 2009/08/28 19:28:00 debdev Exp $"
d706 9
a714 3
  _write_signature(db,TD+'/_temp',role)
  p=subprocess.Popen(GPG_SIGN+['--output',TD+'/_gpg'+role,TD+'/_temp'])
  p.wait()
a715 1
    rmtree(TD)
d717 2
a718 2
  os.system("ar qS "+delta+" "+TD+"/_gpg"+role)
  rmtree(TD)
@


1.184
log
@do_delta() : move signing code inside try: except
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.183 2009/08/28 19:19:08 debdev Exp $"
d331 1
d860 1
a860 1
def do_patch(delta,olddeb,newdeb, info=None, diversions=None):
d863 1
a863 1
    r=do_patch_(delta,olddeb,newdeb,TD=T, info=info, diversions=diversions)
d872 1
a872 1
def do_patch_(delta,olddeb,newdeb, TD, info=None, diversions=None):
d905 1
a905 1
    if DO_GPG:
d907 1
a907 1
    else:
d2498 1
d2500 9
a2508 1
          ret=do_delta(old['File'],new['File'], delta)
d2512 2
d2515 2
a2516 4
            p=open(delta+'-fails','w')
            p.close()
        except KeyboardInterrupt:
          raise
d2518 2
d2521 4
d2528 1
a2528 2
        
        (delta_, percent, elaps, info_delta, gpg_hashes) = ret
d2536 1
a2536 2
            p=open(delta+'-too-big','w')
            p.close()
d2542 2
a2543 1
            pret=do_patch(delta,old['File'],None , info=info_delta)
d2547 1
a2547 2
              p=open(delta+'-fails','w')
              p.close()
d2562 1
a2562 1
        #complete the delta file prepending to it the info
@


1.183
log
@gnupg error as DebDeltaError ;
uniform exception handling in all actions, and exit codes as per man page
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.180 2009/08/28 17:56:58 debdev Exp $"
d1139 5
d1150 1
a1150 7
    (delta, percent, elaps, info, gpg_hashes) = r
    info_hashes=append_info(delta,info)
    if DO_GPG:
      gpg_hashes['info']=info_hashes
      sign_delta(delta,gpg_hashes)

  rmtree(T)
@


1.182
log
@correct bug in DebDeltaError
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.181 2009/08/28 18:25:31 debdev Exp $"
d709 2
a710 2
    print 'GnuPG fails to sign, returncode ',p.returncode,"\n"
    return
d2671 1
a2671 2
      print 'Error!! sources.conf section ',s,'does not contain delta_uri'
      raise SystemExit(1)
d2747 1
a2747 2
    print 'Could not lock dir: ',DEB_DIR, a
    raise SystemExit(1)
d3244 1
a3244 1
      raise SystemExit(1)
d3252 3
a3254 2
    except (KeyboardInterrupt, SystemExit):
        if DEBUG : puke('debpatch exited')
d3256 5
a3260 5
        print  str(s)
        raise SystemExit(1)
    except :
        puke( "Unexpected error" )
        raise SystemExit(1)
d3265 1
a3265 1
    raise SystemExit(1)
d3273 6
a3278 2
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debpatch exited')
d3280 2
a3281 2
    puke( 'debpatch failed',s)
    raise SystemExit(2)
d3287 1
a3287 1
    raise SystemExit(1)
d3292 3
a3294 2
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdeltas exited')
d3296 5
a3300 5
    puke('Failed: ',s)
    raise SystemExit(2)
  except:
    puke('debdelta failed' )
    raise SystemExit(3)
d3306 9
a3314 5
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdeltas exited')
  except:
    puke( 'debdeltas failed')
    raise SystemExit(2)
d3322 9
a3330 5
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdelta-upgrade exited')
  except:
    puke('delta-upgrade failed')
    raise SystemExit(2)
@


1.181
log
@do not use my_popen_read() before it is defined
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.180 2009/08/28 17:56:58 debdev Exp $"
a488 2
  self.retriable = False
  self.exitcode = 2
@


1.180
log
@in case of error , in prepare_for_echo, raise DebDeltaError
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.179 2009/08/28 17:55:00 debdev Exp $"
d243 5
a247 6
if os.path.exists('/proc/cpuinfo'):
  f=my_popen_read('grep bogomips /proc/cpuinfo')
  BOGOMIPS=float(f.read().split(':')[-1])
  f.close()
  del f
else:
@


1.179
log
@add exitcode to DebDeltaError
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.178 2009/08/28 17:53:00 debdev Exp $"
d453 6
a458 7
            print 'Errror in prepare_for_echo.'
            if DEBUG or VERBOSE > 2 :
                print 'string='+repr(s)
                print 'repres='+repr(r)
                print 'shell='+SHELL
                print 'output='+repr(a)
            raise SystemExit(2)
@


1.178
log
@remove dummy()
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.177 2009/08/28 17:51:51 debdev Exp $"
d491 3
a493 1
  def __init__(self,s,retriable=False):
d500 6
@


1.177
log
@fix up some exitcodes:
3 for wrong command line or
4 internal error on some sanity checks
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.176 2009/08/28 17:47:54 debdev Exp $"
a242 4

def dummy(): #otherwise the python mode for emacs fails to index my routines
  pass

@


1.176
log
@move my_popen_read(cmd) ;
def the dummy act()
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.175 2009/08/28 10:30:39 debdev Exp $"
d162 1
a162 1
    raise SystemExit(0)
d178 1
a178 1
      raise SystemExit(2)
d195 1
a195 1
        raise SystemExit(1)
d236 1
a236 1
      raise SystemExit(1)
d241 1
a241 1
    raise SystemExit(1)
d447 1
a447 1
    raise SystemExit(2)
@


1.175
log
@rename _delta_info_unzip_ to _info_patch_unzip_ ; raise a retriable DebDeltaError if 'zlma' or 'python-apt' are missing
@
text
@a105 3
def my_popen_read(cmd):
  return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout

d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.174 2009/08/28 09:08:45 debdev Exp $"
d286 3
d3236 4
@


1.174
log
@remove reference to keyword 'unpack-old', that was obsoleted in RCS 1.6 (rcsdiff -u debdelta)
@
text
@d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.173 2009/08/27 20:51:03 debdev Exp $"
d721 2
a722 1
def _delta_info_unzip_(TD):
d730 2
d744 1
a744 1
  _delta_info_unzip_(TD)
d909 1
a909 1
  _delta_info_unzip_(TD)
d1064 1
a1064 1
            die('This patch needs lzma. Please install the Debian package "lzma".')
d1089 1
a1089 1
        die('This patch needs lzma. Please install the Debian package "lzma".')
d2162 1
a2162 2
      print 'ERROR!!! python module "apt_pkg" is missing. Please install python-apt'
      raise SystemExit
d2703 1
a2703 2
    print 'ERROR!!! python module "apt_pkg" is missing. Please install python-apt'
    raise SystemExit
d2708 1
a2708 2
    print 'ERROR!!! python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12)'
    raise SystemExit
@


1.173
log
@correct bugs in handling of Packages.gz
@
text
@d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.172 2009/08/27 20:46:41 debdev Exp $"
d1045 1
a1045 5
    if 'unpack-old' == a:
      if olddeb == '/':
        die('This patch needs the old version Debian package')
      unpack ('OLD',olddeb,TD)
    elif 'needs-old' == a:
@


1.172
log
@correct typo
@
text
@d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.171 2009/08/27 20:35:41 debdev Exp $"
d2200 1
a2200 1
    assert os.path.basename(packages) == 'Packages'
d2211 1
a2211 1
    if packages[-4:] == '.bz2':
@


1.171
log
@fix some bugs in features handling, add option --disable-feature
@
text
@d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.170 2009/08/27 13:18:08 debdev Exp $"
d2105 1
a2105 1
  script.clos
@


1.170
log
@fix script_zip so that --needsold works again
@
text
@d149 3
d153 1
a153 1
RCS_VERSION="$Id: debdelta,v 1.169 2009/08/27 12:59:13 debdev Exp $"
d178 1
a178 1
                    'signing-key=', "accept-unsigned", "gpg-home=") )
d228 2
d241 6
d1064 2
d1093 2
d1211 5
d1255 7
a1261 1
  info.append('needs-'+USE_DELTA_ALGO)
d1361 1
d1379 1
d1382 1
a2060 1
        info.append('needs-minibzip2')
d2064 1
a2064 1
        info.append('needs-lzma')
d2105 1
a2105 1
  script.close()
d2111 3
a2113 2
  system('lzma -q -9 -k '+v+' PATCH/patch.sh', TD)
  patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.lzma'), 'lzma', 'patch.sh.lzma'))
d2123 2
d2130 1
@


1.169
log
@do_delta_ : do not abuse of short named variables
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.168 2009/08/27 12:13:29 debdev Exp $"
d1038 3
a1040 2
    elif 'needs-old' == a and olddeb == '/':
      die('This patch needs the old version Debian package')
d1343 2
a1344 1
  def script_zip(n,cn,newhead):
d1346 5
a1350 2
      s=prepare_for_echo(newhead)
      script.write("$echo  '"+ s +"' >> "+n+cn +' && ./minigzip -9 < '+n+' | tail -c +'+str(len(newhead)+1)+' >> '+n+cn+' && rm '+n+' \n')
d1352 1
a1352 1
      script.write(' ./minibzip2 -9 < '+n+' >> '+n+cn+' && rm '+n+' \n')
d1354 1
a1354 1
      script.write('lzma -9 < '+n+' >> '+n+cn+' && rm '+n+' \n')
d2057 1
a2057 1
      script.write('cat '+newname+cn+' >> NEW.file ;  rm '+newname+'\n')
@


1.168
log
@adjust indent
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.167 2009/08/27 12:05:31 debdev Exp $"
d1764 1
a1764 1
    if VERBOSE > 2 : print '  finding correspondences  ',n
d1832 1
a1832 1
    if VERBOSE > 2 : print '  scanning ',n
d1986 2
a1987 2
    n = 'NEW/'+name
    system('ar p '+TD+'NEW.file '+name+' >> '+TD+n,TD)
d1989 1
a1989 1
    newsize = os.stat(TD+n)[ST_SIZE]
d2004 1
a2004 1
      p=open(TD+n)
d2007 1
a2007 1
      unlink(TD+n)
d2010 2
a2011 2
      o = 'OLD/'+name
      system('ar p OLD.file '+name+' >> '+o, TD)
d2019 3
a2021 3
      delta_tar(o,n,'CONTROL',skip, chunked_p=False)
      if DEBUG > 3 : md5_ar(TD,n,name)
      script.write('cat '+n+' >> NEW.file ;  rm '+n+'\n')
a2022 2
      o = 'OLD/'+name
      #system('ar p OLD.file '+name+' >> '+o, TD)
d2035 1
a2035 1
      delta_tar(x,n,'DATA',old_conffiles,old_md5,new_md5,\
d2037 3
a2039 2
      if DEBUG > 3 : md5_ar(TD,n,name)
      script.write('cat '+n+' >> NEW.file ;  rm '+n+'\n')
d2041 2
a2042 2
      p=verbatim(n)
      script.write('cat '+p+' >> NEW.file ; rm '+p+'\n')
d2045 10
a2054 9
      o = 'OLD/'+name
      system('ar p OLD.file '+name+' >> '+o, TD)
      script.write('ar p OLD.file '+name+' >> '+o+'\n')
      (o,co) = unzip(o)
      (n,cn) = unzip(n)
      delta_files(o,n)
      script_zip(n,cn)
      script.write('cat '+n+cn+' >> NEW.file ;  rm '+n+'\n')
      unlink(TD+n)
d2065 1
d2084 1
@


1.167
log
@rewrite if's ; delete useless line
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.166 2009/08/27 11:53:00 debdev Exp $"
d1685 1
a1685 1
           raise DebDeltaError(' OSError (at _a_) while writing: '+str(s), True)
@


1.166
log
@Delete useless line
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.165 2009/08/27 11:50:12 debdev Exp $"
d1519 1
a1519 1
    if format ==  FAST :
d1521 1
a1521 1
    if format ==  SLOW :
a1605 1
      penalty=0.2
@


1.165
log
@Do not use full path for xdelta3
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.164 2009/08/26 19:29:06 debdev Exp $"
a1299 1
      pm.close
@


1.164
log
@better print for _gpg
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.163 2009/08/26 19:28:04 debdev Exp $"
d1381 2
a1382 2
      system('/usr/bin/xdelta3 -9 -R -D -n -S djw -s  '+o+' '+n+' '+p,TD,(o,n))
      script.write('/usr/bin/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
@


1.163
log
@debdeltas: add support for Packages.gz   Packages.bz2
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.162 2009/08/24 09:30:58 debdev Exp $"
d885 1
a885 1
        die(delta+": the signature "+a+" fails as follows: "+repr(r))
d888 1
a888 1
        print ' The signature '+a+' is correctly verified for ',delta
@


1.162
log
@correct bug in  1.147 : the extension for bzip2 is .bz2 (!)
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.161 2009/08/22 17:32:32 debdev Exp $"
d2172 7
a2178 1
    for l in open(packages):
d2337 1
a2337 1
    if os.path.basename(alt) == 'Packages':
d2345 1
a2345 1
      if os.path.basename(arg) == 'Packages':
@


1.161
log
@debpatch: do not check, in debug mode, if the output file already exists
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.160 2009/08/22 17:31:15 debdev Exp $"
d2030 1
a2030 1
      elif 'data.tar.bzip2' in ar_list_old :
@


1.160
log
@chech that the signature starts with a standard PGP line
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.159 2009/08/22 17:29:43 debdev Exp $"
a874 2
  if DEBUG and  newdeb and os.path.exists(newdeb) and os.path.getsize(newdeb) > 0 :
      die("Don't want to overwrite: "+newdeb)
@


1.159
log
@rewrite signature parsing, check headers in any order
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.158 2009/08/22 16:59:02 debdev Exp $"
d600 4
@


1.158
log
@in signature , in the "Files:" accept lines starting with any
whitespace; break at the end of this list
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.157 2009/08/22 16:47:39 debdev Exp $"
a635 1
  #should check headers
d638 30
a667 25
  while a and a[:5] != 'Files':
    if a[:5] == "Role:" and a != ("Role: "+role+"\n") :
      return ("ROLE_MISMATCH",a)
    a=f.readline()
  if not a:
    return ("UNCHECKED",dir_list)
  #parse files
  for a in f:
    if a[0] == '\t' or  a[0] == ' ':
      a=a.rstrip('\n')
      a=a.lstrip()
      a=a.split(' ')
      if VERBOSE > 3 : print '    checking hashes ',a
      (md5,sha1,le,na)=a
      if na not in dir_list:
        return ('ABSENT',na)
      (cmd5,csha1,cle)=hashes[na]
      if int(le) != cle:
        return ('SIZE',na)
      # check hashes
      if md5 != cmd5 :
        return ('MD5',na)
      if sha1 != csha1 :
        return ('SHA1',na)
      dir_list.remove(na)
d669 2
a670 1
      break
@


1.157
log
@shorter signature file when creating;
check headers when verifying;
use "maker" for role
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.156 2009/08/22 12:34:52 debdev Exp $"
d647 1
a647 1
    if a[0] == '\t':
d649 1
a649 1
      a=a.lstrip('\t')
d664 2
@


1.156
log
@convert DIR to absolute path, but preserving ending //
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.155 2009/08/22 12:25:50 debdev Exp $"
d600 4
d608 1
a608 1
  r=_verify_signature_no_gpg(p.stdout, DIR)
d624 1
a624 1
def _verify_signature_no_gpg(signature, DIR):
d637 2
d640 2
d671 5
a675 2
  #FIXME write something in 'Signer'
  f.write("Version: 4\nSigner: \nDate: %s\nRole: %s\nFiles: \n" % (time.ctime(),role))
d681 1
a681 1
def sign_delta(delta, db, role="creator"):
@


1.155
log
@correct bug in debdeltas
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.154 2009/08/22 12:25:32 debdev Exp $"
d211 3
a213 1
      DIR = v
@


1.154
log
@do not report output of 'ar x' in debug mode
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.153 2009/08/22 12:17:39 debdev Exp $"
d2487 1
a2487 1
        hashes_info=append_info(delta,info)
@


1.153
log
@move code inside do_delta_ ;
add SystemExit
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.152 2009/08/22 10:57:38 debdev Exp $"
d506 1
a506 1
  if DEBUG and (ignore_output==False and (os.stat(temp_name)[ST_SIZE] > 0 or os.stat(temp_err_name)[ST_SIZE] > 0 )):
d854 1
a854 1
  o=system('ar xvo '+delta,  TD+'/PATCH', return_output=True)
@


1.152
log
@Delete short-named "local" variables, correct bug.
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.151 2009/08/22 10:50:17 debdev Exp $"
d1107 7
a3202 1

d3214 2
a3215 1
  
d3232 2
a3233 7
  else:
    (delta, percent, elaps, info, gpg_hashes) = r
    info_hashes=append_info(delta,info)
    if DO_GPG:
      gpg_hashes['info']=info_hashes
      sign_delta(delta,gpg_hashes)
  
d3242 2
a3243 1
  
d3315 1
a3315 1

@


1.151
log
@Shorter --help s, line suggesting man pages.
@
text
@d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.150 2009/08/22 09:48:41 debdev Exp $"
d241 1
d435 1
d674 1
a674 1
    print 'GPG returncode ',p.returncode," output: \n",a
@


1.150
log
@The full and complete and fantastic GnuPG signature method
@
text
@a18 1
            possible values are: xdelta xdelta-bzip xdelta3 bsdiff
a33 4

            If DIR ends in // , then the dirname of the cmdline argument
            will be appended to DIR, as well (useful when creating archives).
            DIR may also be a Package file, as found in a Debian repository.
a68 2
            (default: /var/cache/apt/archives for root,
              /tmp/archive for non-root users)
a70 7
            it is a comma separated list of (abbreviations of)
             source = debs where there is no /etc/debdelta/sources.conf line
             big = debs where the delta is too big
             error = debs where the delta fails to apply
             unavailable = all other cases
             after = continue downloading debs after patching thread has ended
            default is s,b,e
d86 2
d150 1
a150 1
RCS_VERSION="$Id: debdelta,v 1.149 2009/08/22 07:02:08 debdev Exp $"
@


1.149
log
@Add an option to system() so that the stdout is saved in a separate
file whose name can be returned to the caller
@
text
@d12 2
d29 2
d65 1
d85 1
d98 2
d108 1
a108 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time, traceback, ConfigParser, subprocess
d150 8
d162 1
a162 1
RCS_VERSION="$Id: debdelta,v 1.148 2009/08/21 12:19:07 debdev Exp $"
d178 5
d184 4
a187 2
    ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:' ,
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug') )
d235 7
d340 4
d346 1
d573 119
d864 19
a882 2
  system('ar xo '+delta,  TD+'/PATCH')

d1195 2
d1419 3
d2070 4
d2089 1
a2089 1
  return (delta, percent, elaps, info)
d2454 1
a2454 1
        (delta_, percent, elaps, info_delta) = ret
d2489 7
a2495 1
        append_info(delta,info_delta)
d3236 5
a3240 2
    (delta, percent, elaps, info) = r
    append_info(delta,info)
@


1.148
log
@Do not store informations in patch.sh (it was done for backward
compatibility, but it was Sep 2006)
@
text
@d146 1
a146 1
RCS_VERSION="$Id: debdelta,v 1.147 2009/08/21 10:57:19 debdev Exp $"
d469 3
a471 2
  
def system(a,TD,saveargs=None,ignore_output=False):
d476 2
a477 1
  (temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_system")
d479 1
a479 1
  ret = os.system("cd '" +TD +"' ; ( "+a+" ) > "+temp_name+" 2>&1 ")
d481 1
a481 1
  if DEBUG and (ignore_output==False and os.stat(temp_name)[ST_SIZE] > 0 ):
d484 3
a486 2
      print '  ',repr(i)
  os.unlink(temp_name)
d488 9
a496 1
  if ret == 2:
d498 1
a498 1
  if  ret != 0 and ( ret != 256 or a[:6] != 'xdelta') :
@


1.147
log
@correct bug when the old deb contains data.tar.XX
and the new one contains  data.tar.YY and XX != YY
@
text
@d146 1
a146 1
RCS_VERSION="$Id: debdelta,v 1.146 2009/08/21 10:18:35 debdev Exp $"
a1032 4
  #backward compatibility
  for i in info:
    script.write('#'+i+'\n')

@


1.146
log
@Correct bug in treatment of 'lzma' of stuff inside data.tar ; this
feature is though not tested so far !
@
text
@d146 1
a146 1
RCS_VERSION="$Id: debdelta,v 1.145 2009/08/21 08:16:41 debdev Exp $"
d1825 1
a1825 1
      if name[-3:] == '.gz' :
d1827 2
a1828 2
          return my_popen_read('cd '+TD+'; ar p OLD.file '+name+' | gzip -cd')
      elif name[-4:] == '.bz2' :
d1831 2
a1832 2
          return my_popen_read('cd '+TD+'; ar p OLD.file '+name+' | bzip2 -cd')
      elif name[-5:] == '.lzma' :
d1835 1
a1835 1
          return my_popen_read('cd '+TD+'; ar p OLD.file '+name+' | unlzma -c')
@


1.145
log
@Add new command line tool 'debpatch-url'
@
text
@d146 1
a146 1
RCS_VERSION="$Id: debdelta,v 1.144 2009/08/07 08:15:18 debdev Exp $"
d858 1
a858 1
          system('ar p '+TD+'OLD.file data.tar.lzma | unlzma | tar -x -p -f - -C '+TD+'OLD/DATA', TD)
d1138 1
a1138 1
        script.write('unlzma '+f+'\n')
@


1.144
log
@Use "subprocess" instead of os.popen
@
text
@d82 4
d146 1
a146 1
RCS_VERSION="$Id: debdelta,v 1.143 2009/08/06 21:02:31 debdev Exp $"
d154 1
a154 1
  actions =  ('delta','patch','deltas','delta-upgrade')
d2413 21
a2433 1
  
a2601 19
  def delta_uri_from_config(**dictio):
    secs=config.sections()
    for s in secs:
      opt=config.options(s)
      if 'delta_uri' not in opt:
        print 'Error!! sources.conf section ',s,'does not contain delta_uri'
        raise SystemExit(1)
      match=True
      for a in dictio:
        #damn it, ConfigParser changes everything to lowercase !
        if ( a.lower() in opt ) and ( dictio[a] != config.get( s, a) ) :
          #print '!!',a, repr(dictio[a]) , ' != ',repr(config.get( s, a))
          match=False
          break
      if match:
        return  config.get( s, 'delta_uri' )
    if VERBOSE:
      print '(sources.conf does not provide a server for ', repr(dictio['PackageName']),')'
  
d2825 2
a2826 1
      delta_uri_base=delta_uri_from_config(Origin=origin.origin,
d3067 60
@


1.143
log
@Fix problem when the home directory does not exists (e.g. user "nobody").
@
text
@d96 1
a96 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time, traceback, ConfigParser
d106 3
d142 1
a142 1
RCS_VERSION="$Id: debdelta,v 1.142 2009/08/05 15:39:04 debdev Exp $"
d216 1
a216 1
  f=os.popen('grep bogomips /proc/cpuinfo')
d324 1
a324 1
  p=os.popen('ar t '+f,'r')
d336 1
a336 1
  p=os.popen('tar t '+f,'r')
d386 2
a387 1
    o,i=os.popen2(shell)
d732 1
a732 1
        p=os.popen('env -i dpkg -s '+b)
d771 1
a771 1
    p=os.popen('env -i dpkg -L '+pa)
d1011 1
a1011 1
    p=os.popen('md5sum '+TD+'NEW.file')
d1094 1
a1094 1
      pm=os.popen('md5sum '+TD+n)
d1775 1
a1775 1
    pm=os.popen('cd '+TD+'; ar p OLD.file '+name+' | md5sum -')
d1823 1
a1823 1
          return os.popen('cd '+TD+'; ar p OLD.file '+name+' | gzip -cd')
d1827 1
a1827 1
          return os.popen('cd '+TD+'; ar p OLD.file '+name+' | bzip2 -cd')
d1831 1
a1831 1
          return os.popen('cd '+TD+'; ar p OLD.file '+name+' | unlzma -c')
d1919 1
a1919 1
  f=os.popen('hostname -f')
d2052 1
a2052 1
      p=os.popen('ar p '+f+' control.tar.gz | tar -x -z -f - -O ./control')
@


1.142
log
@Correct indent for HOSTID
@
text
@d139 1
a139 1
RCS_VERSION="$Id: debdelta,v 1.141 2009/08/05 12:12:02 debdev Exp $"
d2319 4
d2324 1
d2326 3
a2328 3
    if not os.path.exists(os.path.dirname(s)):
      print 'Creating: ',os.path.dirname(s)
      os.makedirs(os.path.dirname(s))
d2355 4
a2358 1
    
@


1.141
log
@Support lzma
@
text
@d139 1
a139 1
RCS_VERSION="$Id: debdelta,v 1.140 2009/08/05 10:35:04 debdev Exp $"
d1915 8
a1922 8
    f=os.popen('hostname -f')
    try:
      import hashlib
      HOSTID=hashlib.md5( f.read() ).hexdigest()
    except ImportError:
      import md5
      HOSTID=md5.new( f.read() ).hexdigest()
    f.close()
@


1.140
log
@Use hashlib.md5
@
text
@d139 1
a139 1
RCS_VERSION="$Id: debdelta,v 1.139 2009/08/05 09:14:42 debdev Exp $"
d530 2
d847 1
a847 1
        else:
d849 3
d872 3
d1127 6
d1142 2
d1824 4
d1868 3
d1872 8
a1879 7
  system('gzip -q -9 -n '+v+' PATCH/patch.sh', TD)  
  if  os.path.getsize(TD+'PATCH/patch.sh.gz') > os.path.getsize(TD+'PATCH/patch.sh.bz2') :
    if VERBOSE > 1 : print '  bzip2 wins on patch.sh  '
    patch_append('patch.sh.bz2')
  else:
    if VERBOSE > 1 : print '  gzip wins on patch.sh  '
    patch_append('patch.sh.gz')
@


1.139
log
@Use default for BOGOMIPS if /proc/cpuinfo is not there
@
text
@d96 1
a96 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time, md5, traceback, ConfigParser
a120 5
f=os.popen('hostname -f')
HOSTID=md5.new( f.read() ).hexdigest()
f.close()


d139 1
a139 1
RCS_VERSION="$Id: debdelta,v 1.138 2009/04/08 12:14:46 debdev Exp $"
d1890 10
a1899 1
  
@


1.138
log
@Update copyright.
Move code around.
Add arguments to debdelta-upgrade.
@
text
@a120 4
f=os.popen('grep bogomips /proc/cpuinfo')
BOGOMIPS=float(f.read().split(':')[-1])
f.close()

d144 1
a144 1
RCS_VERSION="$Id: debdelta,v 1.137 2009/04/08 11:46:20 debdev Exp $"
d217 9
@


1.137
log
@Support python-apt 0.7.10. Thanks a lot to Julian Andres Klode
@
text
@d3 1
a3 1
# Copyright (C) 2006-07 Andrea Mennucci.
d64 1
a64 1
Usage: debdelta-upgrade
d148 1
a148 1
RCS_VERSION="$Id: debdelta,v 1.136 2009/03/29 18:52:47 debdev Exp $"
a2274 73
################################################# main program, do stuff

if action == 'patch':
  if INFO  :
    if  len(argv) > 1 and VERBOSE :
      print '(printing info - extra arguments are ignored)'
    elif  len(argv) == 0  :
      print ' need a  filename ;  try --help'
      raise SystemExit(1)
    try:
        delta=abspath(argv[0])
        check_is_delta(delta)
        info=get_info(delta)
        for s in info:
          if s:
            print ' info: ',s
    except (KeyboardInterrupt, SystemExit):
        if DEBUG : puke('debpatch exited')
    except DebDeltaError,s:
        print  str(s)
        raise SystemExit(1)
    except :
        puke( "Unexpected error" )
        raise SystemExit(1)
    raise SystemExit(0)
  #really patch
  if len(argv) != 3 :
    print ' need 3 filenames ;  try --help'
    raise SystemExit(1)


  newdeb=abspath(argv[2])
  if newdeb == '/dev/null':
      newdeb = None

  try:
    do_patch(abspath(argv[0]), abspath(argv[1]), newdeb)
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debpatch exited')
  except Exception,s:
    puke( 'debpatch failed',s)
    raise SystemExit(2)
  
elif action == 'delta' :
  if len(argv) != 3 :  
    print ' need 3 filenames ;  try --help'
    raise SystemExit(1)
  
  delta=abspath(argv[2])
  try:
    r = do_delta(abspath(argv[0]), abspath(argv[1]), delta)
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdeltas exited')
  except DebDeltaError,s:
    puke('Failed: ',s)
    raise SystemExit(2)
  except:
    puke('debdelta failed' )
    raise SystemExit(3)
  else:
    (delta, percent, elaps, info) = r
    append_info(delta,info)
  
elif action == 'deltas' :
  try:
    do_deltas(argv)
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdeltas exited')
  except:
    puke( 'debdeltas failed')
    raise SystemExit(2)
  
  
d2365 1
a2365 1
def delta_upgrade_():
d2738 2
d2932 30
d2963 3
a2965 1
####
d2967 38
a3004 1
if action == 'delta-upgrade':
d3008 1
a3008 1
    delta_upgrade_()
@


1.136
log
@protect against errors in updating the statistics
@
text
@d148 1
a148 1
RCS_VERSION="$Id: debdelta,v 1.135 2008/11/27 12:17:12 debdev Exp $"
d2805 3
d2810 25
a2834 13
    if p.isInstalled and  p.markedUpgrade :
      #thanks a lot to Michael Vogt
      p._lookupRecord(True)
      dpkg_params = apt_pkg.ParseSection(p._records.Record)
      cand = p._depcache.GetCandidateVer(p._pkg)
      deb_path=dpkg_params['Filename']      
      for (packagefile,i) in cand.FileList:
        indexfile = cache._list.FindIndex(packagefile)
        if indexfile:
          deb_uri=indexfile.ArchiveURI(deb_path)
          break
      
      arch=dpkg_params['Architecture']      
a2835 1
      #newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
d2845 5
a2849 5
              ( p.name, p.installedVersion, p.candidateVersion )
      delta_uri_base=delta_uri_from_config(Origin=p.candidateOrigin[0].origin,
                                           Label=p.candidateOrigin[0].label,
                                           Site=p.candidateOrigin[0].site,
                                           Archive=p.candidateOrigin[0].archive,
d2860 2
a2861 2
      delta_name=p.name+'_'+version_mangle(p.installedVersion)+\
                  '_'+ version_mangle(p.candidateVersion)+'_'+\
@


1.135
log
@add support for 'prelink'-ed files , thanks  roman@@khimov.ru
@
text
@d148 1
a148 1
RCS_VERSION="$Id: debdelta,v 1.134 2008/05/03 17:19:10 debdev Exp $"
d2386 4
a2389 1
      self.package_stats[n]=d
d2399 4
a2402 1
      self.upgrade_stats[s]={ 'PatchSpeedRatio' : r }
d2413 4
a2416 1
    self.upgrade_stats[s]=a
@


1.134
log
@review internal DebDeltaError class to avoid problems with python 2.5;
correct typo.
@
text
@d148 1
a148 1
RCS_VERSION="$Id: debdelta,v 1.133 2008/03/16 11:06:07 debdev Exp $"
d672 2
d794 11
a804 2
        if VERBOSE > 3 : print '   symlinking ',divert,' to ',a
        os.symlink(divert, a)
@


1.133
log
@added    debdelta-upgrade --deb-policy
@
text
@d148 1
a148 1
RCS_VERSION="$Id: debdelta,v 1.132 2008/03/10 13:53:02 debdev Exp $"
d450 1
a450 1
    self.args = s
d452 2
a453 6
  def __str__(self):
    if DEBUG:
      if self.retriable:
        return self.args + ' (retriable) '
      else:
        return self.args + ' (non retriable) '
d455 1
a455 1
      return self.args
d459 1
d551 1
a551 1
    raise DebdeltaError('This is not a debdelta file: '+delta)
@


1.132
log
@move debug printing after header
@
text
@d71 9
d142 1
d148 1
a148 1
RCS_VERSION="$Id: debdelta,v 1.131 2008/03/10 13:45:14 debdev Exp $"
d166 1
a166 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt','no-md5','debug') )
d182 1
d2531 3
a2533 1
            no_delta.append( (deb_uri, newdeb) )
d2536 3
a2538 1
            no_delta.append( (deb_uri, newdeb) )
a2550 2
      threads.pop()
      if VERBOSE > 1 : print ' Patching thread ended , bye bye. '
d2553 3
d2614 2
a2651 10
      if not VERBOSE: return False
      if uri[-9:] == '.debdelta':
        conn.request("HEAD", urllib.quote(uri_p[2]+'-too-big'))
        r2 = conn.getresponse()
        r2.read()
        r2.close()
        if r2.status == 200:
          print 'Too big: ',uri
          return False
      _http_whine_(uri,r)
d2821 2
a2822 1
        no_delta.append( (deb_uri, newdeb) )
d2852 8
a2859 1
        no_delta.append( (deb_uri, newdeb) )
d2915 3
a2917 1
      no_delta.append( (deb_uri, newdeb) )
d2930 2
a2931 2
  while threads:
    while no_delta:
d2940 2
a2941 1
    time.sleep(0.2)
@


1.131
log
@capture the stdout and stderr of all system() calls and reformat it
@
text
@d138 1
a138 1
RCS_VERSION="$Id: debdelta,v 1.130 2008/03/10 12:42:50 debdev Exp $"
a920 1
  if DEBUG > 1 : print '  debdelta.conf says we will skip: ', repr(debdelta_conf_skip)
d976 2
@


1.130
log
@support data.tar.bz2
@
text
@d138 1
a138 1
RCS_VERSION="$Id: debdelta,v 1.129 2007/12/27 16:48:59 debdev Exp $"
d454 1
a454 1
def system(a,TD,saveargs=None):
d459 10
a468 2
  ret = os.system("cd '" +TD +"' ; "+a)  
  if DEBUG > 2 : print '    system(',a,')=',ret
d531 2
a532 2
  system('ar x  '+delta+' info info.gz patch.sh patch.sh.gz patch.sh.bz2 2> /dev/null', \
         TD+'/PATCH')
d868 2
a869 1
      system('echo "'+params['NEW/MD5sum']+'  NEW.file" | md5sum -c > /dev/null', TD)
d1828 2
a1829 2
  system('bzip2 --keep -9  '+v+'  PATCH/patch.sh 2>&1', TD)
  system('gzip -9 -n '+v+' PATCH/patch.sh 2>&1', TD)  
@


1.129
log
@do not use variable 'delta' before referencing it
@
text
@d82 1
d138 1
a138 1
RCS_VERSION="$Id: debdelta,v 1.128 2007/12/22 08:01:41 debdev Exp debdev $"
d460 1
d679 1
a800 1

d815 5
a819 1
        system('ar p '+TD+'OLD.file data.tar.gz | tar -x -z -p -f - -C '+TD+'OLD/DATA', TD)
d840 2
d1084 7
a1090 2
    elif  f[-3:] == '.bz2' :
      print 'WARNING ! ',f,' is in BZIP2 format ! please fixme !'
d1098 2
a1099 1
      print 'WARNING ! ',n,' is in BZIP2 format ! please fixme !'
d1772 8
a1779 3
      assert(name[-3:] == '.gz')#should add support for bz2 data.tar
      def x():
        return os.popen('cd '+TD+'; ar p OLD.file '+name+' | gzip -cd')
@


1.128
log
@correctly parse patch option 'needs-bsdiff'
@
text
@d137 1
a137 1
RCS_VERSION="$Id: debdelta,v 1.127 2007/12/21 21:37:00 debdev Exp debdev $"
d2137 2
a2143 2
        delta=os.path.join(deltadirname,deltabasename)
        
@


1.127
log
@do_delta_ reads /etc/debdelta/debdelta.conf
and understands  the option skip
@
text
@d137 1
a137 1
RCS_VERSION="$Id: debdelta,v 1.126 2007/12/17 14:26:35 debdev Exp $"
d831 1
a831 1
    elif 'needs-xdelta3' == a:
@


1.126
log
@add copyright and licence statement
@
text
@d86 1
a86 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time, md5, traceback
d137 1
a137 1
RCS_VERSION="$Id: debdelta,v 1.125 2007/11/15 10:18:24 debdev Exp $"
d889 2
d893 13
d1000 13
a1012 1
    old_conffiles=()
d1400 1
a1400 1
  def delta_tar(old_filename,new_filename,CWD,skip=(),old_md5={},new_md5={}, chunked_p=True):
d1439 7
d1761 2
a1762 1
      delta_tar(x,n,'DATA',old_conffiles,old_md5,new_md5)
d2384 1
a2384 1
  import  thread , pickle, urllib, fcntl, atexit, signal, ConfigParser
@


1.125
log
@debdeltas : if "python-apt" is missing, print a error message and exit
@
text
@d3 3
d137 1
a137 1
RCS_VERSION="$Id: debdelta,v 1.124 2007/11/15 10:12:53 debdev Exp $"
@


1.124
log
@system() : in DEBUG code, correct bug, and make it more robust
@
text
@d134 1
a134 1
RCS_VERSION="$Id: debdelta,v 1.123 2007/11/15 10:06:01 debdev Exp $"
d1797 7
a1803 3
    import apt_pkg
    apt_pkg.InitSystem()
    from apt_pkg import VersionCompare
@


1.123
log
@correct bugs in usage of os.makedirs
@
text
@d134 1
a134 1
RCS_VERSION="$Id: debdelta,v 1.122 2007/11/15 09:14:03 debdev Exp $"
d460 14
a473 6
    if DEBUG and saveargs:
      T=abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
      open(T+'/command','w').write(a)
      for l in saveargs:
        shutil.copy2(l,T)
        s=s+'\n saved argument '+l+' in '+T
@


1.122
log
@if a helper binary differ crashes, save command and arguments in a
temporary dir
@
text
@d134 1
a134 1
RCS_VERSION="$Id: debdelta,v 1.121 2007/11/13 13:47:44 debdev Exp $"
d757 5
a761 4
      if os.path.isfile(divert) and not os.path.islink(divert) :            
        a=os.path.dirname(TD+'/OLD/DATA'+orig)
        if not os.path.exists(a):
          os.makedirs(a)
d2263 2
a2264 2
    if not os.path.exists(s):
      print 'Creating: ',s
a2270 3
      if not os.path.exists(s):
        print 'Creating: ',s
        os.makedirs(os.path.dirname(s))
@


1.121
log
@some code clean up
@
text
@d134 1
a134 1
RCS_VERSION="$Id: debdelta,v 1.120 2007/11/13 13:34:39 debdev Exp $"
d450 1
a450 1
def system(a,TD):
d459 8
a466 1
    die('Error , non zero return status '+str(ret)+' for command "'+a+'"')
d1076 1
a1076 1
      system('/usr/bin/xdelta3 -9 -R -D -n -S djw -s  '+o+' '+n+' '+p,TD)
d1083 1
a1083 1
      system('bsdiff  '+o+' '+n+' '+p,TD)
d1088 2
a1089 2
      system('xdelta delta --pristine --noverify -0 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD)
      system('bzip2 -9 '+p,TD)
d1093 1
a1093 1
      system('xdelta delta --pristine --noverify -9 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD)
@


1.120
log
@debpatch: understands the keywords 'needs-xdelta' and 'needs-bsdiff'
@
text
@d134 1
a134 1
RCS_VERSION="$Id: debdelta,v 1.119 2007/11/13 13:33:25 debdev Exp $"
d1893 2
a1894 1
        if label == 'CMDLINE' :
d1903 1
a1903 1
          os.unlink(f)
d1977 3
a1979 3
  for f in ALT:
    if os.path.basename(f) == 'Packages':
      scan_Packages(f,'ALT')
d1981 1
a1981 1
      ALT_NOP.append(f)
d1984 4
a1987 4
  for f in debs:
    if os.path.isfile(f):
      if os.path.basename(f) == 'Packages':
        for a in  iterate_Packages(f):
d1989 1
a1989 1
          of=info_by_file[a]['Filename']
d1996 2
a1997 2
      elif f[-4: ] != '.deb' :
        print 'Warning: skipping cmd line argument: ',f
d1999 3
a2001 3
        scan_deb(f, 'CMDLINE')
        di=os.path.dirname(f) or '.'
        pa = package_name(f)
d2004 1
a2004 1
          scan_deb_dir(delta_dirname(f,alt), pa, 'ALT')
d2006 3
a2008 3
          scan_delta_dir(delta_dirname(f,DIR), pa)
    elif  os.path.isdir(f) :
      scan_deb_dir(f, None, 'CMDLINE')
d2011 1
a2011 1
          scan_deb_dir(delta_dirname(f,alt), None, 'ALT')
d2013 1
a2013 1
        scan_delta_dir(delta_dirname(f,DIR))
d2015 1
a2015 1
      print 'Warning: '+f+' is not a regular file or a directory.'
d2038 1
a2038 1
    how_many= len( info_pack  )
@


1.119
log
@debdeltas: document the fact that 'Package' files may be used as arguments
@
text
@d134 1
a134 1
RCS_VERSION="$Id: debdelta,v 1.118 2007/11/08 20:24:53 debdev Exp $"
d809 6
@


1.118
log
@get rid of make_parents , use os.makedirs
debdeltas: cache_dir caches both .debs and .deltas
correct some bugs in the previous version:
 extensions are '.deb' and '.delta'
 does not create directories when not needed, but do when needeed
@
text
@d19 1
a19 1
Usage: debdeltas [ option...  ]  [deb_files and dirs]
d29 3
a31 2
            if DIR ends in // , then the dirname of the cmdline argument
            will be appended to DIR, as well (useful when creating archives)
d134 1
a134 1
RCS_VERSION="$Id: debdelta,v 1.117 2007/11/07 10:43:37 debdev Exp $"
@


1.117
log
@debdeltas : can have multiple --alt  options
debdeltas : can parse 'Package' files
@
text
@d133 1
a133 1
RCS_VERSION="$Id: debdelta,v 1.116 2007/10/20 17:14:39 debdev Exp debdev $"
d289 1
a289 13

def make_parents(f):
  assert(f[0] == '/')
  s=f.split('/')
  d=''
  for a in s[:-1] :
    if a:
      d=d+'/'+a
      if not os.path.exists(d):
        os.mkdir(d)
  d=d+'/'+s[-1]
  return d

d750 3
a752 1
        a=make_parents(TD+'/OLD/DATA'+orig)
d1851 2
a1852 1
  def cache_deb_dir(f):
d1854 2
a1855 2
    if f in deb_dir_cache:
      return deb_dir_cache[f]
d1858 2
a1859 1
      if d[-4:] == '.deb':
d1866 1
a1866 1
    deb_dir_cache[f] = cache
d1870 2
a1871 2
    assert( os.path.isdir(f))
    cache = cache_deb_dir(f)
d1909 15
a1925 10
  old_deltas_dir_visited=[]
  def scan_delta_dir(f,pack_filter=None):
    assert( os.path.isdir(f) )
    if f not in old_deltas_dir_visited:
      if pack_filter == None :
        old_deltas_dir_visited.append(f)
      for d in os.listdir(f):
        dt=os.path.join(f,d)
        if os.path.isfile(dt) and ( pack_filter == None or  pack_filter(d) ):
          scan_delta( dt )
d1955 1
a1955 1
        return make_parents(abspath(a)+'/')
d1961 2
a1962 2
  def __name_filter__(n):
    "returns a function that filters by package name"
d1964 3
a1966 4
    n=n.split('_')[0] + '_'
    l=len(n)
    return lambda x : x[:l] == n

d1983 1
d1985 1
a1985 1
            scan_deb_dir(delta_dirname(of,alt), __name_filter__(a), 'ALT' )
d1987 1
a1987 1
            scan_delta_dir(delta_dirname(of,DIR), __name_filter__(a) )
d1993 2
a1994 1
        scan_deb_dir(di, __name_filter__(f), 'SAMEDIR' )
d1996 1
a1996 1
          scan_deb_dir(delta_dirname(f,alt), __name_filter__(f), 'ALT' )
d1998 1
a1998 1
          scan_delta_dir(delta_dirname(f,DIR), __name_filter__(f) )
d2002 2
a2003 1
        scan_deb_dir(delta_dirname(f,alt), None, 'ALT')
a2054 6
    deltadirname=delta_dirname(new['Filename'],DIR)
    free=freespace(deltadirname)
    if free and (free < (newdebsize /2 + 2**15)) :
      if VERBOSE : print 'Not enough disk space for storing ',delta
      continue

d2067 9
a2075 1
        make_parents(abspath(deltadirname)+'/')
d2249 1
a2249 1
    make_parents(s)
d2257 1
a2257 1
      make_parents(s)
@


1.116
log
@convert tabs to spaces
@
text
@d133 1
a133 1
RCS_VERSION="$Id: debdelta,v 1.115 2007/10/17 13:11:46 debdev Exp $"
d192 1
a192 1
      if not os.path.isdir(v):
d1787 1
a1787 1

d1797 91
a1887 1
  deb_dir_visited=[]
a1888 12
  def scan_deb_dir(f, pack_filter, label):
      "pack filter may be a function that matches by basename"
      assert( os.path.isdir(f))
      if f not in deb_dir_visited:
        if pack_filter == None :
          deb_dir_visited.append(f)
        for d in os.listdir(f):
          dt=os.path.join(f,d)
          if os.path.isfile(dt) and d[-4:] == '.deb' and \
                 ( pack_filter == None or  pack_filter(d) ) :
            scan_deb( dt , label )
            
d1912 2
a1913 4
      info_by_file[f]['File'] = of
      pa=info_by_file[f]['Package']
      ar=info_by_file[f]['Architecture']
      ve=info_by_file[f]['Version']
d1915 1
a1915 7
      if pa in avoid_pack and ( avoid_pack[pa]['Version'] == ve ):
        #note that 'f' is in  info_by_file and not in info_by_pack_arch
        if VERBOSE > 1 :     print 'Avoid: ', new['File']
        return
      if  (pa,ar) not in  info_by_pack_arch :
         info_by_pack_arch[ (pa,ar) ]=[]
      info_by_pack_arch[ (pa,ar) ].append( info_by_file[f] )
d1971 8
d1982 10
a1991 1
      if f[-4: ] != '.deb' :
d1993 8
a2000 8
        continue
      scan_deb(f, 'CMDLINE')
      di=os.path.dirname(f) or '.'
      scan_deb_dir(di, __name_filter__(f), 'SAMEDIR' )
      for alt in ALT:
        scan_deb_dir(delta_dirname(f,alt), __name_filter__(f), 'ALT' )
      if CLEAN_DELTAS:
        scan_delta_dir(delta_dirname(f,DIR), __name_filter__(f) )
d2003 1
a2003 1
      for alt in ALT:
d2009 2
d2056 1
a2056 1
    deltadirname=delta_dirname(new['File'],DIR)
@


1.115
log
@debdelta supports multiple --alt options.
@
text
@d133 1
a133 1
RCS_VERSION="$Id: debdelta,v 1.114 2007/10/17 13:00:24 debdev Exp debdev $"
d1822 7
a1828 7
	  p.close()
	  if os.path.getsize(f) == 0 :
	      print ('Warning: '+f+ ' is an empty file; removing it. ')
	      os.unlink(f)
	  else:  
	      print ('Error: '+f+ ' does not seem to be a Debian package ')
	  return
d2282 1
a2282 1
    # synopsis lockf(  	fd, operation, [length, [start, [whence]]])
@


1.114
log
@Recognize 'needs-xdelta3' keyword when patching.
@
text
@d124 1
a124 1
ALT     = None
d133 1
a133 1
RCS_VERSION="$Id: debdelta,v 1.113 2007/08/26 18:20:05 debdev Exp debdev $"
d191 3
a193 3
      ALT = v
      if not os.path.isdir(ALT):
        print 'Error: --alt ',ALT,' does not exist.'
d1910 2
a1911 2
      if ALT:        
        scan_deb_dir(delta_dirname(f,ALT), __name_filter__(f), 'ALT' )
d1916 2
a1917 2
      if ALT:
        scan_deb_dir(delta_dirname(f,ALT), None, 'ALT')
@


1.113
log
@document --delta-algo ; and reformat help
@
text
@d133 1
a133 1
RCS_VERSION="$Id: debdelta,v 1.112 2007/08/26 18:10:48 debdev Exp debdev $"
d815 3
@


1.112
log
@update path of xdelta3 (it is in Debian in /usr/bin/xdelta3)

save delta algo in header
@
text
@d11 4
a14 1
  -M Mb     maximum memory  to use (for 'bsdiff' or 'xdelta')
d35 4
a38 1
  -M Mb     maximum memory to use (for 'bsdiff' or 'xdelta')
d71 4
a74 4
  -v      verbose (can be added multiple times)
 --no-act do not do that (whatever it is!)
  -d      add extra debugging checks
  -k      keep temporary files (use for debugging)
d133 1
a133 1
RCS_VERSION="$Id: debdelta,v 1.111 2007/07/19 12:05:55 debdev Exp debdev $"
@


1.111
log
@backend xdelta3 now refers to xdelta30q
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.110 2007/07/18 12:20:59 debdev Exp debdev $"
d938 2
a1061 3
    #http://code.google.com/p/xdelta/downloads/detail?name=xdelta30m.tar.gz
    #used to crash
    #http://sourceforge.net/tracker/index.php?func=detail&aid=1506523&group_id=6966&atid=106966
d1063 2
a1064 2
      system(' ~/debdelta/xdelta30q/xdelta3 -9 -R -D -n -S djw -s  '+o+' '+n+' '+p,TD)
      script.write(' ~/debdelta/xdelta30q/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
@


1.110
log
@(again) corrected files_similarity_score__noext__() so that it does not eat lists
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.109 2007/07/17 08:09:07 debdev Exp debdev $"
d1064 2
a1065 2
      system(' ~/debdelta/xdelta30m/xdelta3 -9 -s  '+o+' '+n+' '+p,TD)
      script.write(' ~/debdelta/xdelta30m/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
@


1.109
log
@save header of minigzip output in patch script, since it changes in newer versions of zlib
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.108 2007/07/16 07:41:39 debdev Exp debdev $"
d1273 2
a1274 2
        oo.pop()
        nn.pop()
@


1.108
log
@corrected files_similarity_score__noext__() so that it does not eat lists ;
and reviewed correspondence code
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.107 2007/04/13 09:57:31 debdev Exp debdev $"
a988 1
    assert(os.path.isfile(TD+n))
d990 1
d996 1
d1026 1
a1026 1
  def script_zip(n,cn):
d1028 2
a1029 1
      script.write('./minigzip -9 '+n+'\n')
d1405 4
d1629 1
a1629 1
    script_zip(new_filename,new_filename_ext)
d1632 1
a1632 2
  ############ start computing deltas
    
d1652 8
d1695 1
d1704 1
@


1.107
log
@option --debug was incorrectly passed to getopt
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.106 2007/04/13 09:49:03 debdev Exp debdev $"
a1250 1
    "warning: destroys the input"
d1256 2
a1257 2
        oo.pop()
        nn.pop()
d1474 1
a1474 1
        basescore=0.6
d1476 1
a1476 1
        np=file_similarity_premangle(nb)        
d1480 4
a1483 2
          s=files_similarity_score__noext__(op,np) + abs(float(l - nl))/float(l+nl)
          #print ' diff ',s,o
d1487 1
a1487 1
        if oldname and VERBOSE > 2 : print '   best similar  ',int(100*basescore),newname,oldname
@


1.106
log
@code to test  xdelta 30m  and  jojodiff (= jdiff06)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.105 2006/12/21 13:02:11 debdev Exp debdev $"
d145 1
a145 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt','no-md5','--debug') )
@


1.105
log
@add --debug option
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.104 2006/09/07 19:02:57 debdev Exp debdev $"
d1058 2
a1059 1
    #crashes!
d1062 2
a1063 2
      system(' ~/debdelta/xdelta30e/xdelta3 -s  '+o+' '+n+' '+p,TD)
      script.write(' ~/debdelta/xdelta30e/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
d1081 3
@


1.104
log
@fixed stupid grave bug, command 'debdelta' was failing on
   append_info(delta,info,T)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.103 2006/09/07 18:47:01 debdev Exp debdev $"
d145 1
a145 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt','no-md5') )
d152 1
a152 1
    elif o == '-d' : DEBUG += 1
@


1.103
log
@strip / from TMPDIR

correct Predictor so that it really save stats

debdelta-upgrade : close and reopen connection on some errors
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.102 2006/07/23 09:15:04 debdev Exp debdev $"
d2104 1
a2104 1
    append_info(delta,info,T)
@


1.102
log
@do_patch() : add ECHO_TEST in script, to test if using 'echo -ne' or 'echo -n'

prepare_for_echo() : shorter quoted strings

apply_prepare_for_echo() :  check that they work

SHELL is currently /bin/bash

do_patch() : use SHELL in invoking PATCH/patch.sh

----

reviewed exception handling, corrected one error
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.101 2006/07/12 20:14:36 debdev Exp debdev $"
d199 1
a199 1
TMPDIR = os.getenv('TMPDIR') or '/tmp'
d2119 2
d2125 1
a2125 1
      self.dir='/var/lib/debdelta'
d2127 2
a2128 2
      self.dir=os.path.expanduser('~/.debdelta')
    s=os.path.join(self.dir,'upgrade.db')
d2132 1
a2132 1
    self.upgrade_stats=shelve.open(s)
d2134 1
a2134 1
    s=os.path.join(self.dir,'packages_stats.db')
d2136 1
a2136 3
    if not os.path.exists(s) and DEBUG == 0 :
      self.package_stats = None
    else:
d2140 1
a2140 1
      self.package_stats=shelve.open(s)
d2153 1
a2153 1
    if self.package_stats:
d2331 1
a2331 1
  import httplib
d2334 1
d2337 2
a2338 1
  def conn_by_url(url):
d2341 4
d2346 2
d2350 12
a2437 1
      conn=conn_by_url(uri)
d2443 2
d2447 1
a2447 1
      except httplib.HTTPException,e:
d2449 1
a2473 1
      conn=conn_by_url(uri)
d2493 2
d2497 14
a2510 3
      except httplib.HTTPException,e:
        puke( 'Connection error: ',e)
        return e
d2543 1
a2543 1
        if a + 0.5 < time.time() :
d2553 5
a2557 1
      sys.stderr.write("Downloaded:  %s \n" % os.path.basename(uri) )
d2560 1
a2560 1
      return  conn_time , (j+len_downloaded)
a2605 1
      deltas_conn=conn_by_url(delta_uri_base)
d2628 1
a2628 1
      if  isinstance(r, httplib.HTTPException) :
d2676 2
a2677 4
      if r == None or r == False:
        print 'Disappeared ?? ',uri
      elif isinstance(r, httplib.HTTPException) :
        if VERBOSE : print ' You may wish to rerun, to get also: ',uri
d2717 2
a2718 1
    http_conns[i].close()
@


1.101
log
@class Predictor()  : to predict patching time (still unused :-);
 and to store statistics of debdelta_upgrade


this is 'debdelta' package 0.16 in Debian
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.100 2006/07/12 15:10:13 debdev Exp debdev $"
d332 1
d334 1
a334 1
ALLOWED = '<>()[]{}.,;:!_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
d336 18
a353 1
def prepare_for_echo(s):
d355 2
a356 3
  while s:
    a=s[0]
    s=s[1:]
d359 6
d366 3
a368 1
      r += "\\" + ( '0000' +oct(ord(a)))[-4:]
d371 41
d451 1
a451 1
def die(s=None):
a453 1

d484 2
a485 2
  if VERBOSE or e == '':    print s,e,str(typ),str(value)
  else: print s,e
d815 1
a815 1
  system('/bin/sh -e '+a+' PATCH/patch.sh', TD)
d1336 1
d1571 1
a1571 1
      script.write("echo -n -e '"+ s +"'\"$FTH\" >> OLD/mega_cat\n")
d1626 1
a1626 1
    script.write("echo -n -e '"+ s +"' >> NEW.file\n")
d1701 1
a1701 1
      die('internal error')
d1987 1
a1987 2
          print 'KeyboardInterrupt'
          raise SystemExit(2)
d1989 1
a1989 1
          puke( " *** Error while creating delta  "+delta+": ")
d2018 1
a2018 2
            print 'KeyboardInterrupt'
            raise SystemExit(2)
d2020 1
a2020 1
            puke(" *** Error while testing delta  "+delta+": ")
d2059 2
d2065 1
a2065 1
        puke( " Unexpected error: " )
d2080 4
a2083 2
  except DebDeltaError,s:
    puke( 'Failed: ',s)
a2084 5
  except KeyboardInterrupt:
    raise SystemExit(0)
  except:
    puke( 'Failed: ',s)
    raise SystemExit(3)
d2094 2
a2098 2
  except KeyboardInterrupt:
    raise SystemExit(0)
d2100 5
a2104 3
    raise
  (delta, percent, elaps, info) = r
  append_info(delta,info,T)
d2109 4
a2112 2
  except e:
    puke( 'debdeltas failed: ',e)
d2114 1
a2114 1

d2123 1
a2123 1
      self.dir=os.path.expanduser('/var/lib/debdelta')
a2198 8

def delta_upgrade():
  try:
    delta_upgrade_()
  except:
    puke('Failed: ')
    raise SystemExit(2)
  raise SystemExit(0)
d2713 9
a2721 1
  delta_upgrade()
@


1.100
log
@append_info() do_patch()  do_delta() delta_upgrade() :
 wrap up so that they create the /tmp dir, and clean up on return or
 exception


DebDeltaError: define self.args
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.99 2006/07/09 17:02:02 debdev Exp debdev $"
d2051 18
a2068 6
class predictor:
  def __init__():
    s=os.path.expanduser('~/.debdelta/upgrade.db')
    if not os.path.exists(s) and DEBUG < 2 :
      self.upgrade_stats=None
      self.patch_time_predictor=patch_time_predictor_simple
d2073 1
a2073 3
      import shelve
      self.upgrade_stats=shelve.open(s)
      self.patch_time_predictor=patch_time_predictor_math
d2075 10
a2084 2
  def patch_time_predictor_math(p,t=None):
    "Predicts time to patch. If t is given, adapts."
d2086 32
a2117 5
    n=p['NEW/Package']
    d=copy(p)
    d['LocalDeltaTime']=t
    upgrade_stats['Package:'+n]=d
    #
d2124 1
a2124 1
      return patch_time_predictor_simple(p,t)
d2126 4
a2129 15
    if s not in upgrade_stats:
      r=1
      if 'ServerBogomips' in p :
        r=   float(p['ServerBogomips']) / BOGOMIPS
      upgrade_stats[s]={ 'PatchSpeedRatio' : r }
    r=upgrade_stats[s]['PatchSpeedRatio']
    if t:
      nr =  0.8 * r + 0.2 * (  t / ut )
      a=upgrade_stats[s]
      a['PatchSpeedRatio'] = nr
      upgrade_stats[s]=a
      #upgrade_stats.sync()
      #if VERBOSE > 1 :
      #  print ' Upstream ',ut,'PatchSpeedRatio from ',r,' to ',nr
      #  print upgrade_stats[s]['PatchSpeedRatio']
d2215 1
d2217 1
a2217 5
  def patch_time_predictor_simple(p,t=None):
    if 'ServerBogomips' in p and 'PatchTime' in p:
      return (float(p[ 'PatchTime']) / BOGOMIPS * float(p['ServerBogomips']) )
    else:
      return None
d2258 5
a2262 2
              p=patch_time_predictor_simple(p)
              if p and VERBOSE > 1 : print '   (Predicted %.3f sec )'  % p
@


1.99
log
@added some predictor code

other minor changes
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.98 2006/07/09 08:42:36 debdev Exp debdev $"
d274 1
a274 1
def append_info(delta,info,TD):
d276 2
a277 1
  infofile=open(TD+'/PATCH/info','w')
d281 2
a282 1
  system(['ar','rSi','0',delta, 'info'],  TD+'/PATCH')
d371 2
d374 1
a374 1
    self.__str = s
d377 7
a383 1
    return self.__str
d560 13
a572 1
def do_patch(delta,olddeb,newdeb,TD, info=None, diversions=None):
d604 1
d784 11
d796 1
a796 1
def do_delta(olddeb,newdeb,delta,TD):
a1911 1
        T=tempo()          
d1913 1
a1913 1
          ret=do_delta(old['File'],new['File'], delta, T)
a1914 2
          if os.path.exists(delta):
            os.unlink(delta)
a1920 3
          if os.path.exists(delta):
            os.unlink(delta)
          rmtree(T)
d1925 1
a1925 3
          if os.path.exists(delta):
            os.unlink(delta)
        
a1926 1
          rmtree(T)
a1938 1
            rmtree(T)
a1941 2
          rmtree(T)
          T=tempo()
d1944 1
a1944 1
            pret=do_patch(delta,old['File'],None ,T, info=info_delta)
d1949 1
a1949 1
              p.close()              
a1952 1
            rmtree(T)
a1960 1
            rmtree(T)
d1965 1
a1965 2
        append_info(delta,info_delta,T)
        rmtree(T)
a2011 1
  T=tempo()
d2013 1
a2013 1
    do_patch(abspath(argv[0]), abspath(argv[1]), newdeb ,T)
d2015 1
a2015 4
    print 'Failed: ',str(s)
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    rmtree(T)
a2017 3
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    rmtree(T)
d2020 2
a2021 4
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    rmtree(T)
    raise
d2027 1
a2027 2
    
  T=tempo()
d2030 1
a2030 1
    r = do_delta(abspath(argv[0]), abspath(argv[1]), delta ,T)  
d2032 1
a2032 4
    print 'Failed: ',str(s)
    rmtree(T)
    if os.path.exists(delta):
      os.unlink(delta)
d2035 1
a2035 2
    if os.path.exists(delta):
      os.unlink(delta)
a2036 2
    if os.path.exists(delta):
      os.unlink(delta)
a2039 1
  rmtree(T)
d2044 2
a2045 2
  except DebDeltaError,s:
    print 'Failed: ',str(s)
a2098 1
    
d2100 8
d2187 1
a2187 1
    
a2211 1
          T=tempo()
d2213 1
a2213 1
            ret=do_patch(delta,'/',newdeb ,T, diversions=diversions)
a2214 4
            if name in params_of_delta :
              p= params_of_delta[name]
              p=patch_time_predictor_simple(p)
              if p and VERBOSE > 1 : print '   (Predicted %.3f sec )'  % p
a2215 2
            if os.path.exists(newdeb):
              os.unlink(newdeb)
a2216 1
            rmtree(T)
a2219 2
            if os.path.exists(newdeb):
              os.unlink(newdeb)
a2222 2
            if os.path.exists(newdeb):
              os.unlink(newdeb)
d2225 4
a2231 2
          if os.path.exists(T):
            rmtree(T)
@


1.98
log
@ OLD/Size and NEW/Size are the sizes of the .debs (in bytes) ;
 are  saved in .debdelta , and used to check for tmp space and for
 patch results

changed scan_control()
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.97 2006/07/09 08:21:46 debdev Exp debdev $"
d2048 49
d2128 1
a2128 1
  cache.upgrade()
d2427 1
a2427 1
      sys.stderr.write("Downloaded:  %s\n" % os.path.basename(uri) )
d2525 1
a2525 1

d2563 1
d2586 2
@


1.97
log
@do_delta() : never divide control.tar in chunks .. otherwise the first file in the ar will not be '0'
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.96 2006/07/09 07:22:34 debdev Exp debdev $"
d257 1
a257 1
def scan_control(p,params,prefix=None,info=None):
d268 4
a271 3
      i=a.index(':')
      assert(a[i:i+2] == ': ')
      params[prefix+a[:i]] = a[i+2:]
d515 2
d521 1
a521 1
      instsize += int(params['NEW/Size'])
d527 1
a527 1
  if free and free < ( instsize * 1024  ) :
d529 1
a529 1
        ( int(free/1024) , TMPDIR, instsize )
d594 5
d607 1
a607 1
      scan_control(p,dpkg_params,'OLD')
d616 1
a616 1
        if a[:3] == 'OLD' and a != 'OLD/Installed-Size':
d730 6
d775 2
a776 1

a803 1
  params={}
d811 1
a811 1
      scan_control(p,params,o,s)
d817 4
a820 1

d1704 1
a1704 1
      scan_control(p,info_by_file[f])
@


1.96
log
@added some auxiliary routines to get infor from delta
 get_info_slow() get_info_fast() get_info() info_2_db()

then use the info:
  patch_check_tmp_space()
to avoid downloading deltas that cannot be then applied

a first rudimentary
  patch_time_predictor_simple()
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.95 2006/07/07 20:42:36 debdev Exp debdev $"
d591 1
a591 1
  if  DEBUG:
d1227 1
a1227 1
  def delta_tar(old_filename,new_filename,CWD,skip=(),old_md5={},new_md5={}):
d1432 1
a1432 1
      if (a >=  max_chunk_size * chunk_discount) or \
d1434 1
a1434 1
         (a>0 and (a+newtarinfo.size) >= max_chunk_size * chunk_discount ):
d1492 1
d1558 2
a1559 1
      delta_tar(o,n,'CONTROL',skip)
@


1.95
log
@do_patch() :
 -  some auxiliary routines were separated to make code more readable
do_patch() and debdelta-upgrade :
 - do not use 'dpkg -L' that is too slow, scan diversions once and reuse it
@
text
@d82 1
a82 1
from types import StringType, FunctionType, TupleType, ListType
d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.94 2006/07/07 20:24:25 debdev Exp debdev $"
d414 2
d424 5
a428 1
def print_delta_info(delta,TD):
d435 39
a473 4
  info=_scan_delta_info_(TD)
  for s in info:
      print ' info: ',s
      
d495 35
d545 2
d578 1
a578 7
  params={}
  for s in info:
    if ':' in s:
      i=s.index(':')  
      params[s[:i]] = s[i+2:]
    else:
      params[s] = True
d580 3
a582 10
  if 'NEW/Installed-Size' in params and 'OLD/Installed-Size' in params:
    free=freespace(TD)
    if olddeb == '/':
      instsize=int(params['NEW/Installed-Size'])
    else:
      instsize=int(params['NEW/Installed-Size'])+int(params['OLD/Installed-Size'])
    instsize +=  2**14
    if free and free < ( instsize * 1024  ) :
      raise DebDeltaError(' Not enough disk space (%dkB) for applying delta (needs %dkB).' % \
          ( int(free/1024) , instsize ), True )
a1949 1
    T=tempo()
d1953 4
a1956 1
        print_delta_info(delta,T)
a1958 1
        rmtree(T)
a1961 1
        rmtree(T)
a1962 1
    rmtree(T)
d2103 12
a2114 1
      
d2139 4
d2423 4
a2426 1
        print 'Already here: ',abs_delta_name
d2445 11
d2469 8
a2476 1
      
@


1.94
log
@no_delta is now a list of (uri, newdeb)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.93 2006/07/07 12:04:27 debdev Exp debdev $"
d454 16
a469 1
def do_patch(delta,olddeb,newdeb,TD, info=None):
d550 72
d634 1
a634 24
        s=[]
        p=os.popen('env -i dpkg -L '+pa)
        a=p.readline()
        while a:
          a=de_n(a)
          #support diversions
          if a[:26] == 'package diverts others to:':
            continue
          if s and a[:11] == 'diverted by' or  a[:20] == 'locally diverted to:':
            orig,divert=s.pop()            
            i = a.index(':')
            divert = a[i+2:]
            s.append( (orig,divert) )
          else:
            s.append( (a,a) )
          a=p.readline()
        p.close()        
        for orig,divert in s:          
          if os.path.isfile(divert) and not os.path.islink(divert) :            
            a=make_parents(TD+'/OLD/DATA'+orig)
            if VERBOSE > 3 : print '   symlinking ',divert,' to ',a
            os.symlink(divert, a)
          else:
            if VERBOSE > 3 : print '    not symlinking ',divert,' to ',orig
d637 1
a637 15
        def chmod_add(n,m):
          om=S_IMODE(os.stat(n)[ST_MODE])
          nm=om | m
          if nm != om:
            if VERBOSE > 1 : print ' Performing chmod ',n,oct(om),oct(nm)
            os.chmod(n,nm)
        for (dirpath, dirnames, filenames) in os.walk(TD+'OLD/DATA'):
          chmod_add(dirpath,  S_IRUSR | S_IWUSR| S_IXUSR  )
          for i in filenames:
            i=os.path.join(dirpath,i)
            if os.path.isfile(i):
              chmod_add(i,  S_IRUSR |  S_IWUSR )
          for i in dirnames:
            i=os.path.join(dirpath,i)
            chmod_add(i,  S_IRUSR | S_IWUSR| S_IXUSR  )
d1838 1
a1838 1
            pret=do_patch(delta,old['File'],None ,T, info_delta)
d1998 1
d2062 1
a2062 1
            ret=do_patch(delta,'/',newdeb ,T)
@


1.93
log
@revert a patch that should not have been there
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.92 2006/07/07 20:12:56 debdev Exp debdev $"
d2023 1
a2023 1
            no_delta.append(deb_uri)
d2028 1
a2028 1
            no_delta.append(deb_uri)
d2275 1
d2303 1
a2303 1
        no_delta.append(deb_uri)
d2343 1
a2343 1
      no_delta.append(deb_uri)
d2357 2
a2358 2
      uri = no_delta.pop()
      r=download_uri(uri , DEB_DIR+'/'+os.path.basename(uri), debs_down_time, debs_down_size )
@


1.92
log
@always use
  newdeb=os.path.basename(deb_uri)
and not this and other times
  newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.91 2006/07/07 20:08:16 debdev Exp debdev $"
d2302 1
a2302 1
        no_delta.append( (deb_uri, newdeb) )
@


1.91
log
@puke() is the standard reporting of exceptions

httplib.HTTPException are caught (hopefully)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.90 2006/07/07 20:03:14 debdev Exp debdev $"
d2258 2
a2259 1
      newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
@


1.90
log
@delta_uri_from_config() : bug!  was not working
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.89 2006/07/07 12:41:53 debdev Exp $"
d406 6
d1762 1
a1762 4
          (typ, value, trace)=sys.exc_info()
          print " *** Error while creating delta  ",delta,": ",str(typ),str(value)
          if DEBUG :
            print traceback.print_tb(trace)
d1801 1
a1801 4
            (typ, value, trace)=sys.exc_info()
            print " *** Error while testing delta  ",delta,": ",str(typ),str(value)
            if DEBUG > 1:
              print traceback.print_tb(trace)
d1845 1
a1845 1
        print " Unexpected error:",  sys.exc_info()[0]
d2025 1
a2025 4
            (typ, value, trace)=sys.exc_info()
            print " *** Error while applying delta for ",name,": ",str(typ),str(value)
            if DEBUG > 1:
              print traceback.print_tb(trace)
d2133 6
a2138 2
      conn.request("GET", urllib.quote(uri_p[2]),headers=re)
      r = conn.getresponse()
d2148 1
a2148 1
        return False, False
d2181 6
a2186 2
      conn.request("GET", urllib.quote(uri_p[2]),headers=re)
      r = conn.getresponse()
d2294 14
a2307 8
      r,tempname = download_1k_uri(uri,abs_delta_name)
      #print '1K down? ',r and r.status,uri
      if r:
        if r.status == 206:
          a,b,l = _parse_ContentRange(r)
        else:
          l=int(r.getheader('content-length'))
        available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, tempname  ) )
d2309 2
a2310 1
        no_delta.append(deb_uri)
d2326 1
a2326 1
      if r == None:
d2328 3
d2357 3
@


1.89
log
@do_patch() :  newdeb may be None, cannot do os.path.basename(newdeb)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.88 2006/07/07 12:04:27 debdev Exp debdev $"
d2063 1
d2065 4
a2068 1
        if a not in opt or dictio[a] != config.get( s, a) :
d2070 2
a2071 1
      return  config.get( s, 'delta_uri' )
d2073 1
a2073 1
      print 'Warning: sources.conf does not provide a server for', repr(dictio)
@


1.88
log
@properly report problems with sources.conf
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.87 2006/07/07 12:01:56 debdev Exp debdev $"
d603 1
a603 1
      if VERBOSE > 1 : print ' verifying MD5  for ',os.path.basename(newdeb)
d605 1
a605 1
    else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb)
@


1.87
log
@correctly get KeyboardInterrup and exit
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.86 2006/07/07 11:59:07 debdev Exp debdev $"
d2061 1
a2061 1
        print 'Error!! config file section ',s,'does not contain delta_uri'
d2068 1
a2068 1
      print 'Warning: no configured source for', repr(dictio)
@


1.86
log
@do_patch() : warn if MD5 was not verified (when requested)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.85 2006/07/07 11:57:01 debdev Exp debdev $"
d1749 6
d1793 4
d2013 6
@


1.85
log
@rename   check_diff()  to   check_is_delta()
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.84 2006/07/07 11:56:06 debdev Exp debdev $"
d601 3
a603 2
  if DO_MD5 and 'NEW/MD5sum' in params:
      if VERBOSE > 1 : print '  verifying MD5  for ',os.path.basename(newdeb or delta)
d605 1
@


1.84
log
@do_patch() : newdeb may be None
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.83 2006/07/07 11:55:35 debdev Exp debdev $"
d398 1
a398 1
def check_diff(f):
d460 1
a460 1
  check_diff(delta)
d1826 1
a1826 1
        check_diff(delta)
@


1.83
log
@TMPDIR is /tmp or env TMPDIR

now temporary directories are of the form  ${TMPDIR}/debdeltaXXXXXX
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.82 2006/07/06 16:57:36 debdev Exp $"
d602 1
a602 1
      if VERBOSE > 1 : print ' verifying MD5  for ',os.path.basename(newdeb)
@


1.82
log
@'return KeyboardInterrupt' ?? is 'raise KeyboardInterrupt'
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.81 2006/07/06 16:52:03 debdev Exp debdev $"
d199 2
a209 3
    t=os.getenv('TMPDIR')
    if t == None:
      t='/tmp'
d211 1
a211 1
    if a[ : len(t)+4 ] != t+'/tmp' :
d358 1
a358 9
  TD = abspath(tempfile.mkdtemp())
  t=os.getenv('TMPDIR')
  if t == None:
    t='/tmp'
  #this is fascist but still I do not trust my code
  # and if this fails, then __wrap_ fails as well
  if TD[ : len(t) ] != t :
    raise DebDeltaError; ('Sorry I do not like the temp dir "%s"' % TD)
  #
d382 1
a382 4
  t=os.getenv('TMPDIR')
  if t == None:
    t='/tmp'
  if VERBOSE and TD[: (len(t)+4) ] != t+'/tmp' :
@


1.81
log
@DEBUG is now 0

md5 generation and verification is disabled by new option  --no-md5
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.80 2006/06/30 14:20:40 debdev Exp debdev $"
d398 1
a398 1
    return KeyboardInterrupt
@


1.80
log
@debdeltas : ignore (due to disk full) corrupted .debs
 and remove empty ones

debdeltas : forgot a 'old=info_pack[l]' in --clean-alt
@
text
@d9 1
d30 1
d50 1
d66 3
a68 1
  -k      keep temporary files
a70 3
## currently this is always true:
## -d      debug : add md5sums, check that  versions do match

d112 1
a112 1
DEBUG   = 1
d121 2
d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.79 2006/06/30 14:13:12 debdev Exp debdev $"
d145 1
a145 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt') )
d155 1
d505 2
a506 1
    if free and free < ( instsize * 1024 + 2**23 + MAXMEMORY / 6 ) :
d518 1
d609 1
d611 1
d613 2
a614 2
  if DEBUG and 'NEW/MD5sum' in params:
      if VERBOSE > 1 : print ' verifying MD5 ', params['NEW/MD5sum']
d631 2
a632 2
      print ' Patching done, time: %.2fsec, speed: %dkB/sec %s' % \
            (elaps,(debsize / 1024 /  (elaps+.001)),a)
d693 1
a693 1
  if DEBUG:
d1007 1
a1007 1
    if DEBUG > 2 :  script_md5_check_file(o)
d1481 2
a1482 1
  if DEBUG > 1 and newdeb_md5sum :
d1484 1
a1484 3



d1762 1
a1762 1
          if DEBUG>1:
d1784 1
a1784 1
        if DEBUG > 1:
d1800 1
a1800 1
            if DEBUG>1:
d2021 1
a2021 1
            if DEBUG>1:
d2205 1
a2205 1
          sys.stderr.write("%d%% (%4s/s) for ...%s \r" % \
d2208 1
a2208 1
                            uri[-50:]))
d2213 1
a2213 1
      sys.stderr.write("Downloaded: ...%s \n" % uri[-60:])
d2275 1
a2275 1
        print 'Present ',uri,r.status
@


1.79
log
@debdelta-upgrade :
 download_1k_uri()  downloads a small piece of each .debdelta
 then they are sorted by size
 then download_uri()  downloads the rest of  each .debdelta
 (FIXME test_uri is unused now)
@
text
@d123 1
a123 1
RCS_VERSION="$Id$"
d1554 10
d1808 1
@


1.78
log
@debdelta-upgrade : use conf files
 '/etc/debdelta/sources.conf' '~/.debdelta/sources.conf'
 to deduce the source of deltas

debdelta-upgrade :
 use an unique  cache of http connections; changed test_uri() and download_uri()

debdelta-upgrade : since python-apt is only recommended, bail out with a
 nice error if it is missing

Use 'raise SystemExit' instead of sys.exit .

This was uploaded into Debian , package version 0.15.
@
text
@d76 1
a76 1
from stat    import ST_SIZE, ST_MODE, S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR 
d123 4
a2018 3
  threads=[]
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads,no_delta, thread_returns) ) )

d2046 29
d2080 1
a2080 5
      a='Url'
      if uri[-9:] == '.debdelta':
        a='Debdelta'
      s=uri[-60:]
      conn.request("HEAD", urllib.quote(uri_p[2]))
d2082 1
a2085 1
        if VERBOSE > 2: print a,' is present: ',s
d2087 1
a2087 1
      if not VERBOSE: return None        
d2089 1
a2089 1
        conn.request("HEAD", urllib.quote(uri+'-too-big'))
d2094 19
a2112 4
          print a,' is too big: ',s
          return None
      if r.status == 404:
        print a,'is not present: ',s
d2114 15
a2128 2
        print a,'is not available (',repr(r.status), r.reason,'): ', s
      return None
d2135 16
a2150 2
      #should implement Content-Range
      conn.request("GET", urllib.quote(uri_p[2]))
d2152 2
a2153 2
      if r.status != 200:
        if VERBOSE: print 'Not present: ...',uri
d2157 3
a2159 1
      length=r.length
d2162 1
a2162 1
      if free and (free + 2**14 ) < length  :
d2164 2
d2167 10
a2176 2

      out=open(outnametemp,'w')
d2178 4
a2181 4
      conn_time-=a      
      j=0
      s=r.read(min(1024,r.length))
      while s and j < length:
d2187 1
a2187 1
                           (100*j / length,
d2190 1
a2190 1
        s=r.read(min(1024,r.length))
d2196 1
d2199 2
a2200 1

d2204 4
d2252 12
a2263 4
      #download delta
      if not os.path.exists(DEB_DIR+'/'+delta_name):
        if VERBOSE:
          r=test_uri(uri)
d2265 24
a2288 8
          r=True
        if r:
          r=download_uri(uri, DEB_DIR+'/'+delta_name,deltas_down_time,deltas_down_size)
        if r == None:
          no_delta.append(deb_uri)
        else:
          deltas_down_time = r[0]
          deltas_down_size = r[1]
d2291 1
a2291 1
      if os.path.exists(DEB_DIR+'/'+delta_name):
d2293 1
a2293 1
        c=pickle.dumps(  (p.name, DEB_DIR+'/'+delta_name  ,newdeb, deb_uri ) )
d2295 2
@


1.77
log
@bugs corrected in 'debdeltas' :
 do not use variable named 'type'
 do not use 'info' for two different variables
@
text
@a85 3

DELTA_URL="http://tonelli.sns.it/mirror/debian-deltas"

d131 1
a131 1
    sys.exit(0)
d140 1
a140 1
      sys.exit(2)
d155 1
a155 1
        sys.exit(1)
d163 1
a163 1
        sys.exit(3) 
d169 1
a169 1
        sys.exit(3)
d174 1
a174 1
        sys.exit(3)
d179 1
a179 1
        sys.exit(3)
d182 1
a182 1
      sys.exit(0)
d185 1
a185 1
      sys.exit(1)
d367 1
a367 1
class DebDeltaError:
d1811 1
a1811 1
      sys.exit(1)
d1820 1
a1820 1
        sys.exit(1)
d1824 1
a1824 1
        sys.exit(1)
d1826 1
a1826 1
    sys.exit(0)
d1830 1
a1830 1
    sys.exit(1)
d1845 1
a1845 1
    sys.exit(2)
d1850 1
a1850 1
    sys.exit(0)
d1860 1
a1860 1
    sys.exit(1)
d1871 1
a1871 1
    sys.exit(2)
d1888 1
a1888 1
    sys.exit(2)
d1896 1
a1896 1
  import  thread , pickle, urllib, fcntl, atexit, signal
d1898 5
d1905 13
a1917 2

  import  apt, apt_pkg
d1925 1
d1956 1
a1956 1
    sys.exit(1)
a1959 1
  time.sleep(3)
d2021 29
a2049 7
  a=urlparse(DELTA_URL)
  assert(a[0] == 'http')
  deltas_conn=httplib.HTTPConnection(a[1])
  delta_http_base=a[2]

  ###################################### download_uri
  def test_uri(conn,uri):
d2054 1
a2054 1
      conn.request("HEAD", urllib.quote(uri))
d2075 5
a2079 2
  ###################################### test_uri
  def download_uri(conn,uri,outname,conn_time,len_downloaded):
d2082 1
a2082 1
      conn.request("GET", urllib.quote(uri))
d2093 1
a2093 1
        print 'Not enough disk space do download: ',os.path.basename(uri)
d2123 1
a2123 3
    if p.isInstalled and  p.markedUpgrade \
           and p.candidateOrigin[0].origin == 'Debian':

d2133 1
a2133 1
          break      
d2142 1
d2147 11
a2157 2
      
      newdeb = DEB_DIR+'/'+newdeb
d2164 2
a2165 2
      uri=delta_http_base+'/'+os.path.dirname(deb_path)+'/'+delta_name

d2169 1
a2169 1
          r=test_uri(deltas_conn,uri)
d2173 1
a2173 1
          r=download_uri(deltas_conn,uri, DEB_DIR+'/'+delta_name,deltas_down_time,deltas_down_size)
a2185 2
  deltas_conn.close()

d2194 6
a2199 12
  deb_conns={}
  if threads and no_delta:
    if VERBOSE > 1 :
      print ' Downloading deltas done, downloading debs while waiting for patching thread.'
    while threads and no_delta:
      a = no_delta.pop()
      a=urlparse(a)
      assert(a[0] == 'http')
      if a[1] not in deb_conns:        
        deb_conns[a[1]] = httplib.HTTPConnection(a[1])
      r=download_uri(deb_conns[a[1]], a[2] , \
                   DEB_DIR+'/'+os.path.basename(a[2]),debs_down_time, debs_down_size )
a2202 5
    for i in deb_conns:
      deb_conns[i].close()
    
  if VERBOSE > 1 : print ' Downloading done, waiting for patching thread. '
  while threads:
d2204 4
a2207 1

d2343 1
a2343 1
  sys.exit(0)
@


1.76
log
@debdeltas:  better tracebacks in case of errors
 do not exit in case of errors (so, go on creating other deltas)
@
text
@d383 1
a383 1
  if type(a) != type('') :
d1064 1
a1064 1
    if type(oo) == type(''):
d1066 1
a1066 1
    if type(nn) == type(''):
d1521 1
a1521 1
  if AVOID and type(AVOID) == type(''):
d1650 2
a1651 2
    info=info_by_pack_arch[ (pa,ar) ]
    info.sort(order_by_version)
d1653 1
a1653 1
    versions = [ o['Version'] for o in info ]
d1655 1
a1655 1
    versions_not_alt = [ o['Version'] for o in info if o['Label'] != "ALT" ]
d1665 1
a1665 1
    how_many= len( info  )
d1673 1
a1673 1
      new=info[newest]
d1699 1
a1699 1
        old=info[l]
d1741 2
a1742 2
          (type, value, trace)=sys.exc_info()
          print " *** Error while creating delta  ",delta,": ",str(type),str(value)
d1752 1
a1752 1
        (delta_, percent, elaps, info) = ret
d1754 2
a1755 2
        info.append('ServerID: '+HOSTID)
        info.append('ServerBogomips: '+str(BOGOMIPS))
d1770 1
a1770 1
            pret=do_patch(delta,old['File'],None ,T, info)
d1779 2
a1780 2
            (type, value, trace)=sys.exc_info()
            print " *** Error while testing delta  ",delta,": ",str(type),str(value)
d1791 2
a1792 2
          info.append('PatchTime: %.2f' % p_elaps)
        append_info(delta,info,T)
a1796 1
        old=info[l]
d1983 2
a1984 2
            (type, value, trace)=sys.exc_info()
            print " *** Error while applying delta for ",name,": ",str(type),str(value)
@


1.75
log
@debdelta-upgrade: better error message when locking
@
text
@d1101 1
a1101 1

a1739 2
            rmtree(T)
            continue
d1741 4
d1747 1
a1747 3
          rmtree(T)
          raise

a1777 2
              rmtree(T)
              continue
d1779 4
a1782 1
            print " Unexpected error while testing delta:", sys.exc_info()[0]
d1785 2
d1788 2
a1789 1
            raise
d1985 1
d1987 1
a1987 3
              print " Error while applying delta for ",name,": ",str(type),str(value),traceback.print_tb(trace)
            else:
              print " Error while applying delta for ",name,": ",str(type),str(value)
@


1.74
log
@debdelta-upgrade : save in /var/cache/apt/archives if root,
 (and lock it as APT does)
@
text
@a1922 3
    #open("/var/lib/dpkg/lock", O_RDWR|O_CREAT|O_TRUNC, 0640) = 4
    #fcntl64(4, F_SETFD, FD_CLOEXEC)         = 0
    #fcntl64(4, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=0, len=0}) = 0
d1926 1
d1932 2
a1933 3
    if s.errno == 11:
      if DEB_DIR == '/var/cache/apt/archives' :
        print 'Could not lock dir: ',DEB_DIR,  ' (is APT running?)'
d1935 4
a1938 1
      print 'Could not lock dir: ',DEB_DIR, str(s)
d1942 2
a1943 1
  
@


1.73
log
@debdelta : do not complain 'Hmmm... there is a md5 but not a file' for
 conf files
@
text
@d55 3
a57 1
--dir DIR   directory where to save results (default: /tmp/archive)
d1896 1
a1896 1
  import  thread , pickle, urllib
d1910 4
a1913 1
    DEB_DIR='/tmp/archives'
d1916 1
a1916 1
  if not os.path.exists(DEB_DIR):    
d1918 24
a1941 1

a1945 1
  print 'Recreated debs are saved in ',DEB_DIR
d2039 2
d2054 2
a2055 1
      out=open(outname+'.temp','w')
d2074 1
a2074 1
      os.rename(outname+'.temp',outname)
@


1.72
log
@do_patch : print also newdeb (as it did before)
@
text
@d1195 1
a1195 4
        if o not in oldnames:
          #would you believe? 'sql-ledger' contains MD5 for files it does not ship...
          if VERBOSE: print 'Hmmm... there is a md5 but not a file: ',o
        elif o not in skip:
d1197 3
@


1.71
log
@do_patch() : forgot to mkdir OLD/CONTROL
@
text
@d616 5
a620 2
      print ' Patching done, time: %.2fsec, speed: %dkB/sec' % \
            (elaps,(debsize / 1024 /  (elaps+.001)))
@


1.70
log
@debpatch was refactored a bit; now "debpatch --info" works again ;
morover do_patch() will complain if it finds a parameter that is unknown
@
text
@d72 1
a72 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time, md5
d585 8
a592 6
        if olddeb == '/':      
            p=params['OLD/Package']
            for  b in dpkg_keeps_controls :
                a='/var/lib/dpkg/info/' + p +'.'+b
                if os.path.exists(a ):
                    os.symlink(a,TD+'OLD/CONTROL/'+b)
d1949 5
a1953 1
            print " Error while applying delta for ",name,":",sys.exc_info()
@


1.69
log
@debdeltas: one stupid abspath was damaging the --dir /...// behaviour
@
text
@d412 40
d466 1
a466 5
  if INFO:
    system('ar x  '+delta+' patch.sh patch.sh.gz patch.sh.bz2 2> /dev/null', \
           TD+'/PATCH')
  else:    
    if olddeb != '/':
d468 4
a471 5
    if  newdeb and os.path.exists(newdeb) :
      os.rename(newdeb,newdeb+'~')
    system('ar xo '+delta,  TD+'/PATCH')
    
  os.symlink(minigzip,TD+'minigzip')
d473 1
a473 4
  if os.path.exists(TD+'PATCH/patch.sh.gz'):
    system('gunzip PATCH/patch.sh.gz',TD)
  elif os.path.exists(TD+'PATCH/patch.sh.bz2'):
    system('bunzip2 PATCH/patch.sh.bz2',TD)  
d478 2
d482 1
a482 17
    if os.path.isfile(TD+'PATCH/info'):
      #new style debdelta, with info file
      p=open(TD+'PATCH/info')
      info=p.read().split('\n')
      p.close()
      if info[-1] == '': info.pop()
    else:
      #old style debdelta, with info in patch.sh
      p=open(TD+'PATCH/patch.sh')
      s=p.readline()
      s=p.readline()
      while s:
        if s[0] == '#' :
          s=de_n(s)
          info.append(s[1:])
        s=p.readline()
      p.close()
a489 2
    if  INFO :
      print ' info: ',s
d501 1
a501 5
  ## really we will apply the patch
  if not INFO:
    #unpack the old control structure, if available
    os.mkdir(TD+'/OLD/CONTROL')
    if olddeb != '/' :
d503 2
d506 3
a508 4
      system('ar p '+TD+'OLD.file control.tar.gz | tar -x -z -p -f - -C '+TD+'OLD/CONTROL',\
             TD)
    #then we check for the conformance
    if  DEBUG:
d531 3
a533 3
    ###see into parameters: the patch may need extra info
    #as a whole unpack of 'ar' of the old deb
    if 'unpack-old' in params:
d537 1
a537 2

    if 'needs-old' in params and olddeb == '/':
d539 1
a539 2

    if 'old-data-tree' in params :
d584 14
d599 1
a599 13
    if 'old-control-tree' in params and olddeb == '/':      
      p=params['OLD/Package']
      for  b in dpkg_keeps_controls :
        a='/var/lib/dpkg/info/' + p +'.'+b
        if os.path.exists(a ):
          os.symlink(a,TD+'OLD/CONTROL/'+b)

    ##then , really execute the patch
    a=''
    if VERBOSE > 3 : a = '-v'
    system('/bin/sh -e '+a+' PATCH/patch.sh', TD)

    if DEBUG and 'NEW/MD5sum' in params:
d603 1
a603 1
    if newdeb:
d606 2
a607 2
    end_sec = time.time()
    elaps=(end_sec - start_sec)
d609 1
a609 1
    if VERBOSE :
d614 3
a616 3
      print ' Patching done, time: %.2fsec, speed: %dkB/sec, result: %s' % \
            (elaps,(debsize / 1024 /  (elaps+.001)), (newdeb))
    return (newdeb,elaps)
d1805 17
a1821 1
  elif len(argv) != 3 :  
d1824 2
a1825 1
  
d1827 3
d1835 1
a1835 1
    if os.path.exists(newdeb):
d1840 1
a1840 1
    if os.path.exists(newdeb):
d1842 2
d1845 1
a1845 1
    if os.path.exists(newdeb):
d1847 1
a1848 1
  rmtree(T)
@


1.68
log
@.debdeltas start with an info file containing all parameters and also
speed info
@
text
@d1524 3
a1526 3
  def scan_deb(f, label):
      assert( os.path.isfile(f) )
      f=abspath(f)
d1537 1
a1537 1
      info_by_file[f]['File'] = f
@


1.67
log
@debdelta: delta_tar() : do not use  MD5 of files that are really not
  shipped (yes it happens)
@
text
@d72 1
a72 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time
d100 8
d250 1
a250 1
def scan_control(p,params,prefix=None,script=None,stdout = None):
d259 2
a260 2
      if script : script.write('#'+prefix+a+'\n')
      if stdout : stdout.write(' ' + a)
d266 8
d412 1
a412 1
def do_patch(delta,olddeb,newdeb,TD):
a442 2
  #lets scan parameters, to see what it does and what it requires
  params={}
a444 18
  p=open(TD+'PATCH/patch.sh')
  s=p.readline()
  #skip #!/bin/sh
  if s[:2] == '#!' :
    s=p.readline()
  while s :
    if s[0] == '#' :
      s=de_n(s)[1:]
      if VERBOSE > 1 or (VERBOSE and action != 'deltas' and \
                         action != 'delta-upgrade' ) or INFO :
        print ' info: ',s
      if ':' in s:
        i=s.index(':')  
        params[s[:i]] = s[i+2:]
      else:
        params[s] = True
    s=p.readline()
  p.close()
d446 29
d593 3
a600 2
      end_sec = time.time()
      a=(end_sec - start_sec)
d602 2
a603 1
            (a,(debsize / 1024 /  (a+.001)), (newdeb))
d647 1
d654 4
d659 2
a660 8
        s=sys.stdout
        s.write(o+': ')
      else:
        s=None
      p=open(TD+'/'+o+'/CONTROL/control') 
      scan_control(p,params,o,script,s)
      p.close()
      if s: print
d670 1
a670 1
    script.write('#NEW/MD5sum: '+ newdeb_md5sum[:32]+'\n')
d676 1
a676 1
    script.write('#needs-old\n')
d678 6
a683 2
    script.write('#old-data-tree\n')
    script.write('#old-control-tree\n')
d1179 1
a1179 1
          if VERBOSE > 1 : print '  Hmmm... there is a md5 but not a file: ',o
d1454 2
d1458 1
a1469 2

  deltasize = os.stat(delta)[ST_SIZE]
d1471 1
d1473 5
a1477 1
  percent =  deltasize * 100. /  newdebsize 
d1484 1
a1484 1
  return (delta, percent, elaps)
d1711 1
a1711 1
          ret=do_delta(old['File'],new['File'], delta,T)
a1712 1
          #os.chdir(original_cwd)
d1720 2
a1722 1
          #os.chdir(original_cwd)
a1726 2
        #os.chdir(original_cwd)
        rmtree(T)
d1728 10
a1737 6
        if ret and MAX_DELTA_PERCENT:
          deltasize=os.stat(delta)[ST_SIZE]
          if ( ret[1] > (MAX_DELTA_PERCENT+10)  ) or \
            ( ret[1] > (MAX_DELTA_PERCENT+5) and  deltasize >= 150*1024)  or \
            ( ret[1] > (MAX_DELTA_PERCENT-5) and  deltasize >= 1500*1024) or \
            ( ret[1] > (MAX_DELTA_PERCENT-10) and  deltasize >= 15000*1024):
d1742 2
a1743 1
            ret = None
d1745 2
a1746 1
        if DEBUG > 1 and ret :
d1748 1
d1750 1
a1750 1
            do_patch(delta,old['File'],None ,T)
d1758 2
a1761 1
            #os.chdir(original_cwd)
d1766 4
a1769 3
          #os.chdir(original_cwd)
          if os.path.exists(T):
            rmtree(T)
d1823 1
a1823 1
    do_delta(abspath(argv[0]), abspath(argv[1]), delta ,T)  
d1837 2
a1919 1
          #os.chdir(original_cwd)
@


1.66
log
@debdeltas: --clean becomes two options --clean-alt and --clean-deltas

debdeltas: better code
@
text
@d682 1
a682 1
    a=1
a683 1
      a=de_n(f.readline())
d686 1
d1147 4
a1150 1
        if o not in skip:
@


1.65
log
@use     os.path.dirname(f) or '.
since dirname may be empty

print 'Filename in old tar has weird ./ in front'
 instead of  'Weird filename in old tar'   and only with -vvvv

debdeltas: scan also directory where a deb is ;
 distinguish between CMDLINE  debs and non cmdline ;
 create only deltas to newest cmdline deb (from older debs, of course, up -n)
@
text
@d28 1
a28 1
 -n N       how many deltas to produce for each deb package (default 1)
d31 2
a32 1
--clean     delete deltas if newer deb is not in archive
d93 5
a110 1
CLEAN   = False
d112 3
d130 1
a130 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean') )
d140 2
a141 1
    elif o == '--clean' : CLEAN = True
d1501 1
d1503 4
d1565 1
d1576 1
a1576 1
      if CLEAN:
d1582 1
a1582 1
      if CLEAN:
d1593 1
d1595 5
a1599 1
    if CLEAN and (pa,ar) in old_deltas_by_pack_arch :
d1601 1
a1601 1
        if n_d not in versions:
d1606 1
a1606 1
    how_many= len( info  )    
a1608 1
      
d1612 6
a1617 6
    while how_many > 0 :
      new=info[how_many - 1]
      if pa in avoid_pack and ( avoid_pack[pa]['Version'] == new['Version']) :
        if VERBOSE > 1 :   print '(Due to new version) avoid: ', new['File']        
      elif new['Label'] != 'CMDLINE' :
        if VERBOSE > 1 :   print 'Newest version deb was not in cmdline: ', new['File']
d1620 10
a1629 1
      how_many -= 1
d1631 4
a1634 1
    if how_many <= 1 :
d1636 3
a1638 3
    
    l = how_many - 1
    while (l>0) and (l >= how_many - N_DELTAS):
a1640 4

        if pa in avoid_pack and ( avoid_pack[pa]['Version'] == old['Version']):
          if VERBOSE > 1 :     print '(Due to old version) avoid: ', new['File']
          continue
d1644 1
a1644 8
        
        newdebsize=os.path.getsize(new['File'])
        #very small packages cannot be effectively delta-ed
        if newdebsize <= 4 * 1024 :
          #this actually affects 1 every ~60 packages in the archives
          if VERBOSE > 1:     print '  Skip , too small: ', new['File']
          break
        
a1648 1
        deltadirname=delta_dirname(new['File'],DIR)
a1663 5
        free=freespace(deltadirname)
        if free and (free < (newdebsize /2 + 2**15)) :
          if VERBOSE : print 'Not enough disk space for storing ',delta
          break

d1724 11
@


1.64
log
@tested bdelta (=deltup) and  diffball
@
text
@d1089 2
a1090 2
      if VERBOSE > 2 and oldname != de_bar(oldname):
        print ' Weird filename in old tar: ' , oldname 
d1120 2
a1121 2
      if VERBOSE > 2 and newname != de_bar(newname):
        print ' Weird filename in new tar: ' , newname 
d1464 1
a1464 1
  def scan_deb_dir(f,pack_filter=None):
d1474 1
a1474 1
            scan_deb( dt )
d1476 1
a1476 1
  def scan_deb(f):
d1478 1
d1480 4
d1492 1
d1533 1
a1533 1
      f=os.path.dirname(f)
d1556 3
a1558 1
      scan_deb(f)
d1560 1
a1560 1
        scan_deb_dir(delta_dirname(f,ALT), __name_filter__(f) )
d1564 1
a1564 1
      scan_deb_dir(f)
d1566 1
a1566 1
        scan_deb_dir(delta_dirname(f,ALT))
d1571 1
a1571 2

      
d1577 1
d1585 1
a1585 1
      
d1590 18
a1607 5
    if how_many > 1 :
      info.sort(order_by_version)
      l = how_many - 1
      while (l>0) and (l >= how_many - N_DELTAS):
        #os.chdir(original_cwd)
a1609 1
        new=info[how_many - 1]
d1614 3
a1616 4

        if pa in avoid_pack and ( avoid_pack[pa]['Version'] == new['Version']) :
          if VERBOSE > 1 :     print '(Due to new version) avoid: ', new['File']
          break        
@


1.63
log
@debdeltas and debdelta-upgrade : added option --no-act

debdeltas : smarter scanning of --alt dir (faster)
 corrected bug (if abspath() of --dir, the // syntax cannot work!)
@
text
@d736 1
a736 1
    #I   tested bdiff
a737 1
    #It is extremely slow!! and it performs much worse than other choices
d740 2
a741 3
      script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+' ; rm '+p+'\n')
    #
    #I tested zdelta
a742 1
    #it is outperformed by xdelta
d746 11
a756 2
    #
    #I am also testing rdiff
d761 2
a762 2
    #xdelta3 IS buggy and crashes on some inputs
    #xdelta3 crashes!
@


1.62
log
@use abspath also in delta_dirname
@
text
@d106 1
a106 1

d122 1
a122 1
                 ('help','info','needsold','dir=','alt=','avoid=','delta-algo=','max-percent=','clean') )
d131 1
d155 1
a155 1
      DIR = abspath(v)
d160 1
a160 1
      ALT = abspath(v)
d1435 1
a1435 1
def do_deltas(argv):
a1453 4
  debs=[]
  for i in argv:
    debs.append(i)

d1458 12
d1471 2
a1472 7
      if os.path.isdir(f):
        if f not in deb_dir_visited:
          deb_dir_visited.append(f)
          for d in os.listdir(f):
            d=os.path.join(f,d)
            if os.path.isfile(d) and d[-4:] == '.deb' :
              scan_deb( d )
d1488 10
d1499 1
a1499 8
    if os.path.isdir(f):
      if f not in old_deltas_dir_visited:
        old_deltas_dir_visited.append(f)
        for d in os.listdir(f):
          d=os.path.join(f,d)
          if os.path.isfile(d):
            scan_delta( d )
      return
d1526 1
a1526 1
        return make_parents(a)
d1532 7
d1545 2
a1546 2
      if ALT:
        scan_deb(delta_dirname(f,ALT))
d1548 1
a1548 1
        scan_delta(delta_dirname(f,DIR))
d1550 1
a1550 1
      scan_deb(f)
d1552 1
a1552 1
        scan_deb(delta_dirname(f,ALT))
d1554 1
a1554 1
        scan_delta(delta_dirname(f,DIR))
a1565 1

d1570 1
a1570 1
            os.unlink(f_d)
d1605 1
a1605 1
        make_parents(deltadirname+'/')
d1624 4
d1798 1
a1798 2
        (name, delta , newdeb, deb_uri) = pickle.loads(c)      
        if VERBOSE>=2 : print ' Now patching for: ',name
d1800 4
a1803 1
        if True:
@


1.61
log
@--dir and --alt are abspath-ed
@
text
@d1523 1
a1523 1
      return f
@


1.60
log
@debdelta: delta_tar(): only look for correspondence between files with same
 extension (faster)
@
text
@d154 1
a154 1
      DIR = v
d159 1
a159 1
      ALT = v
@


1.59
log
@debdelta(s) -vv : print time  lost in finding correspondences
@
text
@d943 2
a944 1
  def files_similarity_score__(oo,nn):
a946 2
    oo=copy(oo)
    nn=copy(nn)
a947 3
    penalty=0
    if oo.pop() != nn.pop() :
      penalty=0.2
d973 10
a982 1
    return penalty + (l +len(oo) + len(nn)) * 2.0 / float(ln+lo)
d1136 4
a1139 1
      oldnames_premangle[o]=file_similarity_premangle(o)
d1162 2
a1163 1
      if oldname == None:
d1166 3
a1168 2
        np=file_similarity_premangle(newname)
        for o in oldnames:
d1170 1
a1170 1
          s=files_similarity_score__(oldnames_premangle[o],np) + abs(float(l - nl))/float(l+nl)
@


1.58
log
@when printing times, use two digits (that is, centiseconds)
@
text
@d1031 3
d1118 3
d1122 1
d1174 4
a1177 1

d1416 1
a1416 1
    print ' delta time: %.2f sec, speed: %dkB /sec, (%s time: %.2fsec speed  %dkB /sec)' %  \
d1418 1
a1418 1
           USE_DELTA_ALGO,bsdiff_time, bsdiff_datasize / 1024. / (bsdiff_time + 0.001) )
@


1.57
log
@debdelta option --max-percent , useful when testing

option --delta-algo can be 'xdelta' or 'xdelta-bzip' (xdelta with bzip compression)
@
text
@d565 1
a565 1
      print ' Patching done, time: %dsec, speed: %dkB/sec, result: %s' % \
d1388 6
a1393 4
  if  (patchsize > newdebsize / 10 and patchsize > 512 ) or patchsize > 4*1024:
    v=''
    if VERBOSE > 1 : print '  patch.sh is quite large: using bzip2 ' ; v='-v'
    system('bzip2 -9  '+v+'  PATCH/patch.sh', TD)
d1396 1
a1396 3
    v=''
    if VERBOSE > 1 : print '  patch.sh is small: using gzip ' ; v='-v'
    system('gzip -9 -n '+v+' PATCH/patch.sh', TD)  
d1406 1
a1406 1
    print ' delta time: %.1f sec, speed: %dkB /sec, (%s time: %.1fsec speed  %dkB /sec)' %  \
@


1.56
log
@debdelta: better protection against 'disk out of space'  errors
@
text
@d122 1
a122 1
                 ('help','info','needsold','dir=','alt=','avoid=','delta-algo=','clean') )
d134 1
d769 2
a770 2
    elif algo == 'xdelta' :
      system('xdelta delta -n -0 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD)
d774 3
@


1.55
log
@debdelta-upgrade: prints better statistics (in particular with -v )
@
text
@d445 1
a445 1
    if free and free < ( instsize * 1024 + 2**23) :
d589 1
a589 1
  if free and free < newdebsize * 2:
d648 1
a648 1
    if free and free < instsize * 1024  :
d1043 4
a1046 1
        mega_cat.write(a)
d1178 4
a1181 1
        of.write(s)
@


1.54
log
@--delta-algo : option to choose binary delta system (to ease my testings)
@
text
@a1735 1
  len_newdebs=0
d1738 1
d1740 1
a1740 1
  def thread_do_patch(qout,threads,no_delta):
d1742 2
d1753 2
a1754 1
        if True: 
d1770 1
d1776 1
d1779 2
a1780 1
      return len_newdebs
d1783 1
a1783 4
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads,no_delta) ) )

  len_downloaded=0
  conn_time=0
d1857 4
a1860 1
  
d1904 1
a1904 1
          r=download_uri(deltas_conn,uri, DEB_DIR+'/'+delta_name,conn_time,len_downloaded)
d1908 2
a1909 2
          conn_time = r[0]
          len_downloaded = r[1]
a1915 2
        # compute virtual speed . FIXME this is incorrect if patching fails!
        len_newdebs += int( dpkg_params['Size'])
d1925 2
d1936 6
a1941 3
        deb_conns[a[1]] = httplib.HTTPConnection(a[1])      
      download_uri(deb_conns[a[1]], a[2] , \
                   DEB_DIR+'/'+os.path.basename(a[2]),conn_time,len_downloaded )
d1950 21
a1970 6
  if 1 or VERBOSE:
    if conn_time:
      print 'Delta-upgrade download time %dsec speed %s/sec' %\
          (int(conn_time), SizeToStr(len_downloaded / conn_time))
      print '              total time: %dsec; virtual speed: %s/sec.' %  \
          (elaps, SizeToStr(len_newdebs / elaps))
@


1.53
log
@debdeltas: bug! was not scanning cmdline directories anymore

debdelta: code to test bdiff
@
text
@d94 2
a95 4
#seems that 'xdelta' is buggy on 64bit and different-endian machines
USE_XDELTA  = False
#xdelta3 IS buggy and crashes on some inputs
USE_XDELTA3 = False
d122 1
a122 1
                 ('help','info','needsold','dir=','alt=','avoid=','clean') )
d133 1
d732 15
a746 7
      
  def delta_files(o,n):
    " compute delta of two files , and prepare the script consequently"
    nsize = os.path.getsize(TD+n)
    osize = os.path.getsize(TD+o)
    if VERBOSE > 1 : print '  compute delta for %s (%dkB) and %s (%dkB)' % \
       (o,osize/1024,n,nsize/1024)
a747 13
    pp=a_numb_file.next()
    p = 'PATCH/'+pp
    tim = -time.time()

    if DEBUG > 2 :  script_md5_check_file(o)


    #I am also testing bdiff
    if False:
      system(' ~/bin/bdiff -nooldmd5 -nonewmd5 -d  '+o+' '+n+' '+p,TD)
      script.write(' ~/bin/bdiff -p '+o+' '+p+' '+n+'\n')
      tim += time.time()

d749 1
a749 1
    elif False:
d752 2
a753 6

      script.write('rdiff patch '+o+' '+p+' '+n+'\n')
      script_md5_check_file(n)

      tim += time.time()
      
d756 1
a756 1
    elif USE_XDELTA3:
d758 1
a758 3
      script.write(' ~/debdelta/xdelta30e/xdelta3 -d -s '+o+' '+p+' '+n+'\n')
      tim += time.time()

d763 1
a763 4
    elif True: # not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)):    
      if osize > ( 1.1 * (MAXMEMORY / 12))  and VERBOSE  :
        print '  Warning, memory usage by bsdiff on the order of %dMb' % (12 * osize / 2**20)
      global bsdiff_time, bsdiff_datasize
d765 2
a766 4
      tim += time.time()
      bsdiff_time += tim
      bsdiff_datasize += nsize
      script.write('bspatch '+o+' '+n+' '+p+'\n')
d768 1
a768 3
    elif False:
      a=''
      if VERBOSE > 2 : print '    fallback on xdelta instead of bsdiff' ; a = '-v'
d770 2
a771 5
      system('bzip2 -9 '+a+' '+p,TD)
      tim += time.time()
      script.write('bunzip2 '+p+'.bz2\n')
      script.write('xdelta patch '+p+' '+o+' '+n+'\n')
      pp += '.bz2'
d774 1
d776 27
a802 1
    script.write('rm '+p+' '+o+'\n')
d809 1
a809 1
    patch_append(pp)
d1396 1
a1396 1
    print ' delta time: %.1f sec, speed: %dkB /sec, (bsdiff time: %.1fsec speed  %dkB /sec)' %  \
d1398 1
a1398 1
           bsdiff_time, bsdiff_datasize / 1024. / (bsdiff_time + 0.001) )
@


1.52
log
@debdeltas: changed some 'break' in 'continue' (otherwise, with -n > 1,
 it may not create some deltas)
@
text
@d747 7
d755 1
a755 1
    if False:
d1433 1
a1433 1
              scan_delta( d )
@


1.51
log
@debdeltas: added  '--alt DIR' option
@
text
@d1532 1
a1532 1
          break        
d1538 1
a1538 1
        newdebsize=os.stat(new['File'])[ST_SIZE]
d1555 2
a1556 1
          break
d1559 1
a1559 1
          break
d1563 1
a1563 1
          break
@


1.50
log
@delta_files() does not unzip files any more

 mega_cat_chunk():
  patching is done in background
  chunk size increases progressively
@
text
@d22 6
a27 2
            if DIR ends in // , recreate directory tree
            (i.e. the dirname of deb_file will be used as well)
d105 1
d124 1
a124 1
                 ('help','info','needsold','dir=','avoid=','clean') )
d158 5
d1417 1
d1420 8
d1449 1
a1449 1
            scan_delta( os.path.join(f,d))
d1469 8
a1476 5
  def delta_dirname(f):
    "compute delta dirname given deb path"
    if DIR:
      if DIR[-2:] == '//' :
        a=DIR+os.path.dirname(f)
d1479 1
a1479 1
        return DIR
d1481 1
a1481 1
      return os.path.dirname(f)
d1489 2
d1492 1
a1492 2
        di=delta_dirname(f)
        scan_delta(di)
d1494 5
a1498 6
      a=None
      for a in  filter( lambda a : a[-4:] == '.deb' ,os.listdir(f) ) :
        scan_deb(os.path.join(f,a))
      if CLEAN and a:
        di=delta_dirname(os.path.join(f,a))
        scan_delta(di)
d1549 1
a1549 1
        deltadirname=delta_dirname(new['File'])
@


1.49
log
@debdelta-upgrade: if a delta fails to apply, then queue the
 corresponding .deb for download
@
text
@d724 1
a724 1
  def delta_files(o,n,unzip_p=True):
d730 1
a730 6
    if unzip_p:
      (o,co) = unzip(o)
      (n,cn) = unzip(n)
    else:
      co=''
      cn=''
a732 7
    ## according to the man page,
    ## bsdiff uses memory equal to 17 times the size of oldfile
    ## but , in my experiments, this number is more like 12
    #free=freespace(TD)
    #if free == None :
    #  free = MAXMEMORY * 16
    #    #this interferes with chhnked  and (osize < (free / 8)) :
d754 4
a757 1
    #bsdiff is sooooo slow!
a780 1
    script_zip(n,cn)
d1005 1
a1005 1
  ##########################################################################
d1144 1
a1144 1
      p = 'PATCH/tmp_new_tar'
d1156 8
a1163 2
      delta_files('OLD/mega_cat',p)
      script.write('cat '+p+' >> '+new_filename+'; rm '+p+'\n')
d1179 3
d1190 3
a1192 3
      if (a >= MAXMEMORY / 12 ) or \
         (a >= MAXMEMORY / 13 and one_old_file ) or \
         (a>0 and (a+newtarinfo.size) >= MAXMEMORY / 12):
d1199 1
a1199 1
          if mega_cat.tell() >= MAXMEMORY / 12:
d1206 1
d1257 1
d1259 2
d1333 2
d1336 2
a1337 1
      script.write('cat '+n+' >> NEW.file ;  rm '+n+'\n')
@


1.48
log
@correction to similarity fun

debpatch: add write permission to files, otherwise 'rm ' in script
will choke
@
text
@d1687 1
d1689 3
d1699 1
a1699 1
  def thread_do_patch(qout,threads):
d1708 1
a1708 1
        (name, delta , newdeb) = pickle.loads(c)      
d1714 1
a1714 1
            if VERBOSE == 0 : print 'Created ',newdeb
d1719 1
d1724 1
d1736 1
a1736 1
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads) ) )
a1748 4

  #these are the packages that do not have a delta
  no_delta = []

d1859 1
a1859 1
          no_delta.append(deb_uri   )
d1867 1
a1867 1
        c=pickle.dumps(  (p.name, DEB_DIR+'/'+delta_name  ,newdeb) )
@


1.47
log
@faster similarity engine (difflib is too slow)
@
text
@d524 1
a524 1
              chmod_add(i,  S_IRUSR   )
d937 1
a937 1
      while oo[-1] == nn[-1]:
d941 1
a941 1
      while oo[0] == nn[0]:
@


1.46
log
@correct --help on -M

option --allow-xdelta is no more (will put it in config file)

added some code to test xdelta3 : unusable! it is too buggy!

added some code to test rdiff

script_md5_check_file() to check MD5 of partial files while patching
and
debdelta -ddd will do that

properly remove files in delta_files()

solved bug: md5 correspondence was using old conf files

 all parameters are at the beginning of patch.sh
@
text
@d71 1
d926 1
d928 37
d973 1
@


1.45
log
@show speed (=slowness) of bsdiff
@
text
@d10 1
a10 1
  -M Mb   maximum memory (to decide if using 'bsdiff' or 'xdelta')
d26 1
a26 1
  -M Mb     maximum memory to use for 'bsdiff' or 'xdelta'
d54 1
a54 1
doc_common="""
d90 3
a92 1
ALLOW_XDELTA = False
d118 1
a118 1
                 ('help','info','needsold','allow-xdelta','dir=','avoid=','clean') )
a127 1
    elif o == '--allow-xdelta' :  ALLOW_XDELTA = True
d597 1
a597 12
  
  if DEBUG:
    # compute a MD5 of NEW deb
    p=os.popen('md5sum '+TD+'NEW.file')
    a=p.readline()
    p.read()
    p.close
    new_md5sum=a[:32]
    script.write('#NEW/MD5sum: '+ new_md5sum[:32]+'\n')
  else:
    new_md5sum=None
  
d616 17
d680 10
d745 22
a766 1
    if not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)):    
d775 2
a776 2
      script.write('rm '+p+' '+o+'\n')
    else:
a783 1
      script.write('rm '+p+' '+o+'\n')
d786 3
d914 1
d980 1
d983 1
d987 1
d1060 2
a1061 1
        reverse_old_md5[old_md5[o]] = o
d1146 3
d1156 4
a1159 1
          _append_("OLD/"+CWD+"/"+one_old_file)
d1219 1
a1219 1

a1224 6
  if NEEDSOLD :
    #this delta needs the old deb 
    script.write('#needs-old\n')
  else:
    script.write('#old-data-tree\n')
    script.write('#old-control-tree\n')
d1304 2
a1305 2
  if DEBUG > 1 and new_md5sum :
    script.write('echo "'+new_md5sum+'  NEW.file" | md5sum -c > /dev/null')
d1417 3
@


1.44
log
@really correct (!) bug that should have been fixed in previous version:
"debdeltas: create dir before checking for freespace in it!"
@
text
@d563 5
a567 1

d708 2
a709 2
    nsize = os.stat(TD+n)[ST_SIZE]
    osize = os.stat(TD+o)[ST_SIZE]
d727 1
d731 1
d733 3
d743 1
d751 1
a751 1
    deltasize = os.stat(TD+p)[ST_SIZE]
d753 2
a754 1
      print '   delta is %3.2f'  % ( deltasize * 100. /  nsize ) , '% of ',n
d1283 3
a1285 2
    print ' delta time: %dsec, speed: %dkB per second ' %  \
          (elaps, newdebsize / 1024 / (elaps+0.001))
d1293 1
d1449 1
a1449 1
        if free and (free < (newdebsize /2 + 1024)) :
d1509 1
@


1.43
log
@debdeltas: create dir before checking for freespace in it!
@
text
@d1421 1
a1421 1
        os.makedirs(deltadirname)
@


1.42
log
@debdeltas: can --clean unusable debdeltas
@
text
@d1420 3
a1422 1
        delta=os.path.join(delta_dirname(new['File']),deltabasename)        
d1435 1
a1435 1
        free=freespace(os.path.dirname(delta))
@


1.41
log
@less verbosity on delta of .gz files.
@
text
@d21 4
a24 4
            (otherwise they go in the dir of the newer deb)
            if DIR ends in // , then the dirname of deb_file
            the  will be used as well
 -n N       how many deltas to produce for each deb (default 1)
d26 2
a27 2
  -M Mb     maximum memory (to decide if using 'bsdiff' or 'xdelta',
            and how much memory to use for 'xdelta' )
d99 1
a99 1

d116 1
a116 1
                 ('help','info','needsold','allow-xdelta','dir=','avoid=') )
d125 1
d305 7
a311 1

d1316 41
d1360 3
d1364 1
d1366 4
a1369 1
        scan_deb(f+'/'+a)
d1372 1
d1378 11
a1388 1
    how_many= len( info_by_pack_arch[ (pa,ar) ] )    
d1390 1
a1390 1
      print '   I see: ',pa,[ o['Version'] for o in info_by_pack_arch[(pa,ar)]]
d1393 1
a1393 1
      info_by_pack_arch[ (pa,ar) ].sort(order_by_version)
d1398 2
a1399 2
        old=info_by_pack_arch[ (pa,ar) ][l]
        new=info_by_pack_arch[ (pa,ar) ][how_many - 1]
d1419 3
a1421 9
        if DIR:
          if DIR[-2:] == '//' :
            a=DIR+os.path.dirname(new['File'])+'/'+deltabasename
            delta=make_parents(a)
          else:
            delta = DIR+'/'+deltabasename
        else:
          delta = os.path.dirname(new['File']) + '/'+deltabasename
          
@


1.40
log
@do_delta(): report only weirdest permissions
@
text
@d665 1
a665 1
      print '  appending ',f,' of size ', a,' to debdelta, %3.2f'  % ( a * 100. /  newdebsize ) , '% of new .deb'
d789 1
d792 3
a794 2
    if ( os.path.getsize(TD+n) - before + 10 ) < 200 :
      if VERBOSE > 1: print '   Not worthwhile gunzipping: ',n
d842 2
a843 2
      if i == pack_level and VERBOSE > 2:
        print '   Warning: wrong guess to re-gzip to equal file: ',gzip_flags,r,n
d845 1
a845 1
      if VERBOSE > 1: print '  Warning: cannot re-gzip to equal file: ',r,n
d851 2
a852 2
    if VERBOSE > 1 :
      print '  ',n,': ',
@


1.39
log
@do_delta(): adjustments to treatment of gzipped files
@
text
@a972 1
      a=("%o" % newtarinfo.mode)
d974 5
a978 3
      if VERBOSE and (( t == '2' and a  != '777' ) or \
                      ((t == '5' or t == '0') and (a not in [ '755' , '644' ]))):
        print ' Weird permission: ',newname,a,repr(newtarinfo.type)
@


1.38
log
@corrected bug!  gunzip cannot be called on symlinks! patching failed
on debs from filesystem!
@
text
@a760 1
      s=oa
d763 1
a763 2
        s+=oa
      #print repr(s)
d773 1
a774 3
    l=10
    oa=of.read(2)
    na=nf.read(2)    
d790 4
d849 2
@


1.37
log
@do_delta(): move parameters in  patch.sh at beginning
@
text
@d847 3
a849 3
    system("gunzip '"+o+"'",TD)
    script.write("gunzip '"+o+"'\n")
    delta_files(o[:-3],p+'.new')
d851 1
a851 1
    script.write("mv "+p+".new '"+o[:-3]+"' ; gzip "+gzip_flags+" '"+o[:-3]+"'\n")
@


1.36
log
@do_delta(): new code to gunzip .gz files in data.tar before delta, and gzip after
@
text
@a736 1
      #redundant ', %3.2f'  % ( deltasize * 100. /  newdebsize ) , '% of new .deb'
d1154 3
a1156 1

d1194 1
a1194 1
    elif  name[:11] == 'control.tar' :
a1195 1
      script.write('#old-control-tree\n')
a1206 1
      script.write('#old-data-tree\n')
@


1.35
log
@do_debdelta(): use a generator to generate successive names for files
@
text
@d736 2
a737 2
      print '   delta is %3.2f'  % ( deltasize * 100. /  nsize ) , '% of ',n,\
            ', %3.2f'  % ( deltasize * 100. /  newdebsize ) , '% of new .deb'
a741 1
    unlink(TD+n)
d743 112
d1061 1
a1061 1
      os.unlink(p)
d1117 2
d1120 7
a1126 1
      mul=len( old_used[oldname]) > 1 #multiple usage
d1140 1
d1225 1
@


1.34
log
@do_debdelta(): delta_tar(): ignore empty files
@
text
@d574 8
a581 4
  #counter for numbered files  FIXME this will not work in threads....
  global deltacount
  deltacount = 0 

d670 1
a670 3
    global deltacount
    deltacount += 1
    pp=str(deltacount)
d707 1
a707 3
    global deltacount
    deltacount += 1
    pp=str(deltacount)
d936 1
a936 3
      global deltacount
      deltacount += 1
      p = 'PATCH/'+ str(deltacount)
d950 1
@


1.33
log
@debdelta_upgrade: thanks to Michael Vogt, now uses APT caches at best
@
text
@d837 2
a838 2
             not oldtarinfo.isreg():
        continue      
d892 3
d999 4
@


1.32
log
@do_delta(): delta_tar(): chunk tar , always use 'bsdiff'
@
text
@a79 1
DEB_URL="http://ftp.debian.org/debian"
a1391 25

  #this is slow but currently python-apt does not provide these info
  f=os.popen('apt-cache dumpavail')
  #f=open('/var/lib/dpkg/available')
  dpkg_avail={}
  parse_dist(f,dpkg_avail)
  f.close()

  ## FIXME : how do I get the URI and/or architecture out of python-apt ??
  def fake_uri(p):
    if p.sourcePackageName[:3] == 'lib':
      b=p.sourcePackageName[:4]
    else:
      b=p.sourcePackageName[0]
    c=p.candidateOrigin[0].component
    # rely on usual pool structure....
    return c+'/'+b+'/'+p.sourcePackageName
  def get_uri_from_dpkg(p):
    ##this sucks... it is soooo slow..., but it works
    dpkg_params={}
    pip=os.popen('env -i dpkg -p '+p.name)
    parse_dist(pip,dpkg_params)
    pip.close()
    assert(dpkg_params['Version'] == p.installedVersion )
    return dpkg_params
a1480 1
      shorturi=uri[-50:]
d1484 3
a1486 1
        if VERBOSE: print 'Not present: ...',s
d1490 4
d1507 1
a1507 1
                            shorturi))
d1512 1
a1512 1
      sys.stderr.write("Downloaded: ...%s \n" % shorturi)
d1520 11
a1530 5
      
      dpkg_params=dpkg_avail[p.name]
      if 'Filename' not in dpkg_params :
        if VERBOSE: print ' I am lost ! apt-cache dumpavail  does not give URI for ',p.name
        continue
a1532 1
      deb_uri=dpkg_params['Filename']
d1543 1
a1543 1

d1551 1
a1551 1
      uri=delta_http_base+'/'+os.path.dirname(deb_uri)+'/'+delta_name
d1573 1
a1573 4
        if 'Size' in dpkg_params:
          len_newdebs += int( dpkg_params['Size'])
        elif VERBOSE:
          print ' (Size of %s will not be counted in virtual speed ...)' % p.name
d1583 2
a1584 1
  if  threads and no_delta:
a1586 4
    a=urlparse(DEB_URL)
    assert(a[0] == 'http')
    debs_conn=httplib.HTTPConnection(a[1])
    debs_http_base=a[2]
d1588 9
a1596 5
      a=no_delta.pop()
      download_uri(debs_conn, debs_http_base+'/'+a , \
                   DEB_DIR+'/'+os.path.basename(a),conn_time,len_downloaded )
    
    debs_conn.close()
@


1.31
log
@do_patch(): add permission to files in old data.tar ('angband' does
 not have permissions for the user to read files)
@
text
@d82 1
d90 3
d117 1
a117 1
                                   ('help','info','needsold','dir=','avoid=') )
a121 1

d126 1
d128 7
a134 1
    elif o == '-M' :    MAXMEMORY = 1024 * 1024 * int(v)
d424 5
a428 2
    instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size'])
    if free and free < instsize * 1024  :
a695 1
    if VERBOSE > 1 : print '   compute delta for  ',o,' and ',n
d698 2
d713 7
a719 4
    free=freespace(TD)
    if free == None :
      free = MAXMEMORY * 16
    if ( osize < (MAXMEMORY / 12)) and (osize < (free / 8)) :
d851 3
a853 1
    
d855 1
d930 20
a949 1
    if VERBOSE > 2 : print '   scanning ',n
d952 13
a964 9
    for oldname in oldnames :
      if (oldname in skip) or (oldname in old_used ) :
        continue
      if VERBOSE > 2 : print '   provide also old file ', oldname
      #mega_cat.write(fake_tar_2nd)
      #script.write("echo -n -e \"$FTH\" >> OLD/mega_cat\n")
      _append_( "OLD/"+CWD+"/"+oldname )
      if mega_cat.tell() > 2**21 :
        break
d967 16
d1007 1
d1012 4
a1015 1
      delta_files('OLD/mega_cat',new_filename)
d1050 1
a1050 1
    if VERBOSE > 1: print '  studying ' , name , ' of len ' , newsize
@


1.30
log
@do_delta(): less disk space needed for computing delta
@
text
@d69 1
a69 1
from stat    import ST_SIZE
d426 1
a426 1
      system('ar p '+TD+'OLD.file control.tar.gz | tar -x -z -f - -C '+TD+'OLD/CONTROL',\
d491 16
a506 2
        system('ar p '+TD+'OLD.file data.tar.gz | tar -x -z -f - -C '+TD+'OLD/DATA', TD)

@


1.29
log
@debpatch: support for local diversions
@
text
@d71 3
a574 1
        print o
d576 1
d589 1
a589 1
    instsize=int(params['NEW/Installed-Size']) + 2 * int(params['OLD/Installed-Size'])
d772 1
d775 1
a787 1
      unlink(TD+w)
d790 1
a792 3
    
    (old_filename,old_filename_ext) = unzip(old_filename,False)
    (new_filename,new_filename_ext) = unzip(new_filename)
d795 7
a801 1
    oldtar = tarfile.open(TD+old_filename, "r")
d809 1
a809 1
      if VERBOSE and oldname != de_bar(oldname):
d816 6
a821 1

d836 1
a836 1
      if VERBOSE and newname != de_bar(newname):
a843 1
    multiple={}
d890 1
a890 3
      if oldname in old_used:
        multiple[oldname]=True
      else:
d897 12
d929 2
a930 3
      mul=multiple.get(oldname)
      if VERBOSE > 2 :  print '   adding reg file: ', oldname, mul
      oldtar.extract(oldtarinfos[oldname],TD+"OLD/"+CWD )
d932 1
a932 8

    #there may be files that have been renamed and edited...
    for oldname in oldnames :
      if (oldname in skip) or (oldname in old_used ) :
        continue
      if VERBOSE > 2 : print '   provide also old file ', oldname
      oldtar.extract(oldtarinfos[oldname],TD+"OLD/"+CWD )
      _append_( "OLD/"+CWD+"/"+oldname )
d935 1
a935 1
    if os.path.exists(TD+'OLD/mega_cat'):#now, always
d937 1
a941 1
    unlink(TD+old_filename)
d1007 5
a1011 2
      system('ar p OLD.file '+name+' >> '+o, TD)
      delta_tar(o,n,'DATA',old_conffiles,old_md5,new_md5)
@


1.28
log
@MAX_DELTA_PERCENT = 70

--needsold in debdeltas

def tempo(): fascist check

class DebDeltaError:  some errors (e.g. not enought disk space) are retriable

do_delta(): first simply scan for md5 ;
 delta_tar(): then in a double pass, use them, otherwise filename
   similarity, AND properly check for multiple use of old files (!)

 def shell_not_allowed(name): encodes my paranoia for shell quoting

 def file_similarity(): compute filename similarity

debdelta_upgrade(): test_uri() before downloading
  show % of download
@
text
@d468 5
a472 3
          if s and a[:11] == 'diverted by':
            orig,divert=s.pop()
            #support diversions
@


1.27
log
@MAX_DELTA_PERCENT = 60

avoid FutureWarning

Readjusted verbosity  :-> .
Now -vvv is really verbose, gives per-file info.

append(s) is renamed to append_NEW_file(s)

fake_tar_header_2nd() and delta_tar() :
  write some part of the tar header into mega_cat :
  really improves compressibility with xdelta, for .deb with many
  small files

delta_tar() : do not use system('cat')

debdelta-upgrade : do not fork , use real threads.

debdelta-upgrade : while the thread is patching,
 also download some .debs for which deltas are not available.
@
text
@d82 1
a82 1
MAX_DELTA_PERCENT = 60
d120 1
a120 1
    elif o == '--needsold' and action == 'delta' :  NEEDSOLD = True
d167 1
d295 8
d312 1
a312 1
  def __init__(self,s):
d314 1
d413 2
a414 2
      die(' Not enough disk space (%dkB) for applying delta (needs %dkB).' % \
          ( int(free/1024) , instsize ) )
d537 1
a537 1
    die('Error: not enough disk space in '+TD)
d543 1
a543 1
  #counter for numbered files
d581 1
a581 36

  # uses MD5 to detect identical files (even when renamed)
  data_identical={}
  if os.path.exists(TD+'/NEW/CONTROL/md5sums') and \
     os.path.exists(TD+'/OLD/CONTROL/md5sums') :
    f=open(TD+'/OLD/CONTROL/md5sums')
    do={}
    a=de_n(f.readline())
    while a:
      m , n = a[:32] ,  de_bar( a[34:] )
      do[ m ] = n
      a=de_n(f.readline())
    f.close()
    f=open(TD+'/NEW/CONTROL/md5sums')
    dn={}
    mul={}
    us={}
    r=None
    a=de_n(f.readline())
    while a:
      m , n = a[:32] ,  de_bar( a[34:] )
      if m in do:
         #sometimes there are multiple occurences of the same file...
         # ( as in    tetex-doc_3.0-18_all.deb    )
         r=do[m]
         if r in us:
           mul[r] = True
         us[ r ] = True
         dn[ n ] = r
      a=de_n(f.readline())
    f.close()
    for n in dn:
      data_identical[ n ] = ( dn[ n ], mul.get( dn[ n ] ) )
    del f,dn,do,a,mul,n,m,r,us

    
d584 1
a584 1
    instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size'])
d586 2
a587 2
      die(' Not enough disk space (%dkB) for creating delta (needs %dkB).' % \
          ( int(free/1024) , instsize ) )
d589 2
a590 1
  ## check for conffiles 
d598 27
d711 24
d767 1
a767 1
  def delta_tar(o,n,w,skip=(),renames={}):
d787 23
a809 7
    (o,co) = unzip(o,False)
    (n,cn) = unzip(n)
    oldtar = tarfile.open(TD+o, "r")
    oldnames = oldtar.getnames() #map( de_bar , oldtar.getnames() )
    oldused={}
    if VERBOSE > 2 : print '   scanning ',n
    newtar = tarfile.open(TD+n, "r")
d811 2
a812 2
      name = de_bar( newtarinfo.name )

d817 65
a881 1
        print ' weird permission: ',name,a,repr(newtarinfo.type)
d883 4
a890 2
      #mega_cat.write(newtarinfo.name)
      #script.write("echo -n -e '"+ s +"' >> OLD/mega_cat\n")
d900 4
a903 14
      multiple = False
      if name in renames:
        ( oldname , multiple ) = renames[name]
      else:
        oldname = name

      if   oldname in skip :
        if VERBOSE > 2 : print '   skip using old file ', name
        continue

      if '"' in oldname or "'" in oldname or '\\' in oldname or '`' in oldname :
        #FIXME should use it , by properly quoting for the shell script
        if VERBOSE  : print ' weird chars in old file ', oldname
        continue
d905 4
a908 13
      if oldname in oldnames  :
        oldtarinfo = oldtar.getmember(oldname)        
        assert( oldtarinfo.name == oldname )
        if oldtarinfo.isreg() :
          if VERBOSE > 2 :
            if name in renames : print '   use identical old file ', oldname
            else:  print '   use same name old file ', oldname
          oldused[oldname] = name
          del name
          oldtar.extract(oldname,TD+"OLD/"+w )
          _append_( "OLD/"+w+"/"+oldname , not multiple)
        elif VERBOSE > 2 : print '   not regular in old : ', name
      elif VERBOSE > 2 : print '   not present in old : ', name
d912 5
a916 7
      if (oldname not in oldused) and  (oldname not in skip) :
        oldtarinfo = oldtar.getmember(oldname)
        assert( oldtarinfo.name == oldname )
        if oldtarinfo.isreg() :
          if VERBOSE > 2 : print '   provide also old file ', oldname
          oldtar.extract(oldname,TD+"OLD/"+w )
          _append_( "OLD/"+w+"/"+oldname )
d920 2
a921 2
      rmtree(TD+'/OLD/'+w)
      delta_files('OLD/mega_cat',n)
d923 4
a926 4
      p=verbatim(n)
      script.write('mv '+p+' '+n+ '\n')
    unlink(TD+o)
    script_zip(n,cn)
d992 1
a992 1
      delta_tar(o,n,'DATA',old_conffiles,data_identical)
d1107 1
a1107 1
        new=info_by_pack_arch[ (pa,ar) ][l+1]
d1143 4
a1156 6
        except KeyboardInterrupt:
          #os.chdir(original_cwd)
          if os.path.exists(delta):
            os.unlink(delta)
          rmtree(T)
          raise
d1163 3
d1170 1
d1177 1
a1177 1
          if ( ret[1] > (MAX_DELTA_PERCENT+10) and  deltasize >= 15*1024 ) or \
a1190 6
          except KeyboardInterrupt:
            #os.chdir(original_cwd)
            if os.path.exists(delta):
              os.unlink(delta)
            rmtree(T)
            raise
d1193 5
a1197 3
            #os.chdir(original_cwd)
            if os.path.exists(delta):
              os.unlink(delta)
d1278 1
a1278 1
  import  thread , pickle
a1366 1
  global conn_time
a1377 4
  a=urlparse(DEB_URL)
  assert(a[0] == 'http')
  debs_conn=httplib.HTTPConnection(a[1])
  debs_http_base=a[2]
d1383 30
a1412 4
  def download_uri(conn,uri,outname):
      if VERBOSE > 2: print ' uri ',uri
      conn.connect()
      conn.request("GET", uri)
d1415 1
a1415 3
        if VERBOSE: print '  Uri is not available',uri,repr(r.status), r.reason
        data1 = r.read()
        #conn_time+=time.time()
d1417 2
a1418 1
      ext=string.split(outname,'.')[-1]
a1420 1
      global conn_time
d1423 2
a1424 2
      s=r.read(1024)
      while s:
d1429 2
a1430 2
          sys.stderr.write("%s %s ( %s/sec ) for %s \r" % \
                           (ext,SizeToStr(j),
d1432 2
a1433 2
                            delta_name[:50]))
        s=r.read(1024)
d1435 1
a1435 1
      conn.close()
d1437 1
a1437 1
      sys.stderr.write(" " * 70 + '\r')
d1439 1
a1439 1
      return j
d1457 1
a1457 1
        if VERBOSE: print  'Already downloaded: ',newdeb
d1460 1
a1460 1
      if VERBOSE:
d1474 7
a1480 2
      if not os.path.exists(DEB_DIR+'/'+delta_name):        
        r=download_uri(deltas_conn,uri, DEB_DIR+'/'+delta_name)
d1484 2
a1485 1
          len_downloaded += r
d1492 5
d1498 1
a1498 4
      if 'Size' in dpkg_params:
        len_newdebs += int( dpkg_params['Size'])
      elif VERBOSE:
        print ' (Size of %s will not be counted in virtual speed ...)' % p.name
d1506 14
a1519 5
  while threads and no_delta:
    a=no_delta.pop()
    download_uri(debs_conn, debs_http_base+'/'+a , \
                 DEB_DIR+'/'+os.path.basename(a) )

@


1.26
log
@safer code: rmtree, rmdir and unlink check that they are working in
temp dir

safer code: avoid inserting into shell code filenames with ' or " or \

code does not rely on CWD, so it is thread safe now.

shorter patch.sh, by using CR macro, and bzip

MD5 is now added as a parameter, so it is up to 'debpatch' to check it

corrected buggy 'break' when 'I am lost' in debdeltas
@
text
@d76 3
d82 1
a82 1
MAX_DELTA_PERCENT = 50
d94 2
d150 1
a150 1
    if VERBOSE : print ' would unlink ',a
d152 1
a152 1
    if VERBOSE : print ' would rmdir ',a
d154 1
a154 1
    if VERBOSE : print ' would rm -r ',a
d320 1
a320 1
    print 'Warning system in ',TD,' for ',a
d332 1
a332 1
    die('Error: '+f+ 'does not seem to be a Debian package ')
a359 1
    #os.chdir(TD+'/PATCH')
d388 1
a388 1
      if VERBOSE > 2 or (VERBOSE and action != 'deltas' and \
d474 1
a474 1
            if VERBOSE > 4 : print '    not symlinking ',divert,' to ',orig
d484 1
a484 1
          os.symlink(a,'OLD/CONTROL/'+b)
d488 1
a488 1
    if VERBOSE > 4 : a = '-v'
d492 1
a492 1
      if VERBOSE: print ' veryfing MD5 ', params['NEW/MD5sum']
a551 5
  def append(s):
    'appends some data to NEW.file'
    s=prepare_for_echo(s)
    script.write("echo -n -e '"+ s +"' >> NEW.file\n")

d559 1
a559 1
      if  VERBOSE > 1 :
a570 2
  ## helper sh function for script, for delta_tar()
  script.write('CR () { cat "$@@"  >> OLD/mega_cat ; rm "$@@" ;}\n')
d625 3
a627 2
    if VERBOSE > 3 :
      print '   appending ',f,' of size ', os.stat(TD+'PATCH/'+f)[ST_SIZE]
d636 1
a636 1
    if VERBOSE > 3 : print '   including "',name,'" verbatim in patch'
d659 1
a659 1
  def delta_files(o,n):
d661 1
a661 1
    if VERBOSE > 3 : print '   compute delta for  ',o,' and ',n
d664 6
a669 2
    (o,co) = unzip(o)
    (n,cn) = unzip(n)
d683 1
d685 5
a689 2
      if VERBOSE > 3 : print '   fallback on xdelta instead of bsdiff' 
      system('xdelta delta -n -9 -m'+str(MAXMEMORY)+'M '+o+' '+n+' '+p,TD)
d691 3
a693 2
    ## clean up
    script.write('rm '+p+' '+o+'\n')
d697 3
a699 2
    if VERBOSE > 2 :
      print '  delta is  %3.4f'  % ( deltasize * 100. /  nsize ) , '% of ',n
d705 32
d738 2
d745 2
a746 1
    #helper fun
d748 6
a753 1
      system("cat '"+w+"' >>  OLD/mega_cat", TD)
d765 1
a765 1
    if VERBOSE > 3 : print '   scanning ',n
d769 18
d788 1
a788 1
        if VERBOSE > 4 : print '  not regular in new : ', name
d790 1
d798 1
a798 1
        if VERBOSE > 4 : print '   skip using old file ', name
d801 3
a803 2
      if '"' in oldname or "'" in oldname or '\\' in oldname :
        if VERBOSE > 3 : print '   weird chars in old file ', oldname
d810 3
a812 3
          if VERBOSE > 4 :
            if name in renames : print '  use identical old file ', oldname
            else:  print '  use same name old file ', oldname
d817 2
a818 2
        elif VERBOSE > 4 : print '  not regular in old : ', name
      elif VERBOSE > 4 : print '  not present in old : ', name
d826 1
a826 1
          if VERBOSE > 4 : print ' provide also old file ', oldname
d829 3
a831 2
    
    if os.path.exists(TD+'OLD/mega_cat'):
d841 4
d858 1
a858 1
  append(s)
d869 1
a869 1
    if VERBOSE > 2: print ' studying ' , name , ' of len ' , newsize
d872 1
a872 1
    if VERBOSE > 4: print '  ar line: ',repr(s)
d874 1
a874 1
    append(s)
d884 1
a884 1
      append( p.read(newsize))
d920 1
a920 1
      append(extrachar)
d924 2
a925 2
    if VERBOSE > 2: print '  ar leftover character: ',repr(s)
    append(s)
a927 1
    if VERBOSE > 2 : print '   ',a
d934 3
a936 2
    if VERBOSE > 2 : print '  patch.sh is quite large: using bzip2 '
    system('bzip2 -9  PATCH/patch.sh', TD)
d939 3
a941 1
    system('gzip -9 -n PATCH/patch.sh', TD)  
d951 2
a952 3
    if VERBOSE :
      print '  delta time: %dsec, speed: %dkB per second ' %  \
            (elaps, newdebsize / 1024 / (elaps+0.001))
d960 2
d1009 2
a1010 2
    if VERBOSE>3:
      print ' I see: ',pa,[ o['Version'] for o in info_by_pack_arch[(pa,ar)]]
d1016 1
a1016 1
        os.chdir(original_cwd)
d1022 1
a1022 1
          if VERBOSE :     print '(Due to old version) avoid: ', new['File']
d1026 1
a1026 1
          if VERBOSE :     print '(Due to new version) avoid: ', new['File']
d1031 3
a1033 3
        if newdebsize <= 8 * 1024 :
          #this actually affects 1 every ~30 packages in the archives
          if VERBOSE > 1:     print 'Skip , too small: ', new['File']
d1049 1
a1049 1
          if VERBOSE > 2:     print 'Skip , already exists: ',delta
d1052 1
a1052 1
          if VERBOSE > 2:     print 'Skip , tried and too big: ',delta
d1066 1
a1066 1
          os.chdir(original_cwd)
d1072 1
a1072 1
          os.chdir(original_cwd)
d1078 1
a1078 1
          os.chdir(original_cwd)
d1082 1
a1082 1
        os.chdir(original_cwd)
d1102 1
a1102 1
            os.chdir(original_cwd)
d1109 1
a1109 1
            os.chdir(original_cwd)
d1114 1
a1114 1
            os.chdir(original_cwd)
d1119 1
a1119 1
          os.chdir(original_cwd)
d1190 2
a1193 5
  original_cwd = os.getcwd() 
  import httplib
  conn=httplib.HTTPConnection("tonelli.sns.it")
  delta_http_base='/mirror/debian-deltas'

d1213 2
a1214 1
  f=open('/var/lib/dpkg/available')
d1243 1
a1243 1
  #qoutf=os.fdopen(qout)
d1245 1
a1245 1
      if VERBOSE>=2 : print ' Patching thread started. '
d1253 1
a1253 1
        (name,tmp_delta , newdeb) = pickle.loads(c)      
d1255 1
a1255 3
        # my fault...  do_patch() wants its own CWD!
        pid=os.fork()
        if pid == 0:
d1258 1
a1258 1
            ret=do_patch(tmp_delta,'/',newdeb ,T)
d1268 4
a1271 3
          if os.path.exists(tmp_delta):
            unlink(tmp_delta)
          os.chdir(original_cwd)
a1273 4
          #exit fork
          return
        else:
          os.waitpid(pid,0)
d1275 1
a1275 1
      if VERBOSE>=2 : print ' Patching thread ended , bye bye. '
d1281 1
d1284 52
d1343 1
a1343 1
        if VERBOSE: print ' I am lost ! "dpkg -p ',p.name,'" does not give URI!'
d1368 7
a1374 30
      if VERBOSE > 2: print ' uri ',uri

      conn.connect()
      conn.request("GET", uri)
      r = conn.getresponse()
      if r.status != 200:
        if VERBOSE: print '  delta is not available',repr(r.status), r.reason
        data1 = r.read()
        #conn_time+=time.time()
        continue
      a=time.time()
      conn_time-=a
      (delta_fd, tmp_delta) = tempfile.mkstemp()
      j=0
      s=r.read(1024)
      while s:
        len_downloaded += len(s)
        j+=len(s)
        os.write(delta_fd,s)
        if a + 0.5 < time.time() :
          a=time.time()
          sys.stderr.write(" %s ( %s/sec ) for %s \r" % \
                           (SizeToStr(j),
                            SizeToStr(len_downloaded/(a+conn_time)),\
                            delta_name))
        s=r.read(1024)
      os.close(delta_fd)
      conn.close()
      conn_time+=time.time()
      sys.stderr.write(" " * 70 + '\r')
d1376 5
a1380 3
      #append to queue
      c=pickle.dumps(  (p.name,tmp_delta,newdeb) )
      os.write(qin, c + '\t' )
d1389 10
a1398 1
  if VERBOSE>=2 : print ' Downloading done, waiting for patching thread. '
d1409 4
a1412 2
##################################################### apt method
### still work in progress
d1417 4
a1420 1
      
@


1.25
log
@debdeltas: '-n N' to decide how many deltas to compute for each package.
@
text
@d26 2
a27 1
  -M Mb     maximum memory (to decide if using 'bsdiff' or 'xdelta')
a70 1
from os      import unlink, rmdir
d151 7
a157 1
  def rmtree(a):
d159 1
a159 1
      shutil.rmtree(a)
d161 8
a168 1
      print ' Warning! when trying to remove ',repr(a),'got OSError',repr(str(s))
d173 1
a265 9
def unpack(d,f,T):
  "unpacks 'ar' file f in directory d"
  assert(os.path.exists(f))
  cwd = os.getcwd()
  os.chdir(T+'/'+d)
  system('ar xo '+f)
  os.chdir(cwd)


d308 1
a308 1
def system(a):
d311 6
a316 1
  ret = os.system(a)
d340 4
a343 1
def do_patch(delta,olddeb,newdeb,TD):  
d355 3
a357 2
    os.chdir(TD+'/PATCH')
    system('ar x  '+delta+' patch.sh patch.sh.gz patch.sh.bz2 2> /dev/null')
d363 1
a363 3
    unpack ('PATCH',delta,TD)
  #from here on, we live in the temp dir
  os.chdir(TD)
d365 1
a365 1
  os.symlink(minigzip,'minigzip')
d367 4
a370 4
  if os.path.exists('PATCH/patch.sh.gz'):
    system('gunzip PATCH/patch.sh.gz')
  elif os.path.exists('PATCH/patch.sh.bz2'):
    system('bunzip2 PATCH/patch.sh.bz2')  
d374 1
a374 1
  if not os.path.isfile('PATCH/patch.sh'):
d376 1
a376 1
  p=open('PATCH/patch.sh')
d409 2
a410 1
      system('ar p OLD.file control.tar.gz | tar -x -z -f - -C OLD/CONTROL')
d418 1
a418 1
        p=open('OLD/CONTROL/control')
d472 1
a472 1
        system('ar p OLD.file data.tar.gz | tar -x -z -f - -C OLD/DATA')
d484 7
a490 2
    if VERBOSE >= 4 : a = '-v'
    system('/bin/sh -e '+a+' PATCH/patch.sh')
d492 1
a492 1
      shutil.move('NEW.file',newdeb)
d507 3
d520 1
a520 1

a528 3
  #from here on, we live in the temp dir
  os.chdir(TD)

d534 1
a534 1
  script=open('PATCH/patch.sh','w')
d536 12
a547 2


d556 1
a556 1
      os.mkdir(TD+'/'+o+'/CONTROL')
d558 1
a558 1
      system('ar p '+o+'.file control.tar.gz | tar -x -z -f - -C '+o+'/CONTROL')
d571 4
d628 4
a631 4
    os.chdir(TD+'/PATCH')
    system(['ar','qSc', delta,f])
    unlink(f)
    os.chdir(TD)
d639 1
a639 1
    os.rename(f,p)
d646 1
a646 1
      system('gunzip '+f)
d664 2
a665 2
    nsize = os.stat(n)[ST_SIZE]
    osize = os.stat(o)[ST_SIZE]
d679 1
a679 1
      system('bsdiff  '+o+' '+n+' '+p)
d683 1
a683 1
      system('xdelta delta -n -9 '+o+' '+n+' '+p)
d689 1
a689 1
    deltasize = os.stat(p)[ST_SIZE]
d695 3
a697 5
    unlink(o)
    unlink(n)
    if DEBUG:
      pass #implement MD5

d700 1
a700 1
    if os.path.exists('OLD/mega_cat'):
d702 2
a703 2
      os.unlink('OLD/mega_cat')
    
d705 2
a706 3
      system("cat '"+w+"' >> OLD/mega_cat")
      unlink(w)          
      script.write("cat '"+w+"'  >> OLD/mega_cat\n")
d708 3
a710 1
        script.write("rm '"+w+"'\n")
d714 1
a714 1
    oldtar = tarfile.open(o, "r")
d718 1
a718 1
    newtar = tarfile.open(n, "r")
a726 1
        if VERBOSE > 4 : print '   identical!  ', oldname, name, multiple
d733 4
d742 3
a744 1
          if VERBOSE > 4 : print '  use old file ', oldname
d747 1
a747 1
          oldtar.extract(oldname,"OLD/"+w )
d759 1
a759 1
          oldtar.extract(oldname,"OLD/"+w )
d761 3
a763 3
          
    if os.path.exists('OLD/mega_cat'):
      rmtree('OLD/'+w)
d768 1
a768 1
    unlink(o)
d788 2
a789 2
  ar_list_old= list_ar('OLD.file')
  ar_list_new= list_ar('NEW.file')
d793 1
a793 1
    system('ar p NEW.file '+name+' >> '+n)
d795 1
a795 1
    newsize = os.stat(n)[ST_SIZE]
d810 1
a810 1
      p=open(n)
d813 1
a813 1
      unlink(n)
d818 1
a818 1
      system('ar p OLD.file '+name+' >> '+o)
d821 1
a821 1
      for a in os.listdir('OLD/CONTROL') :
d830 1
a830 1
      system('ar p OLD.file '+name+' >> '+o)
d839 1
a839 1
      system('ar p OLD.file '+name+' >> '+o)
d854 1
a854 7
  if DEBUG:
    # add a MD5 check to script
    p=os.popen('md5sum NEW.file')
    a=p.readline()
    p.read()
    p.close
    a=de_n(a)
d856 1
a856 1
    script.write('echo "'+a+'" | md5sum -c > /dev/null')
d860 2
a861 2
  patchsize = os.stat('PATCH/patch.sh')[ST_SIZE]
  if  patchsize > newdebsize / 5 and patchsize > 512 :
d863 1
a863 1
    system('bzip2 -9  PATCH/patch.sh')
d866 1
a866 1
    system('gzip -9 -n PATCH/patch.sh')  
d1121 3
d1222 2
a1223 1
        break
@


1.24
log
@Rewrite code  delta_tar()  , use  _append_() .

Properly exit from forks!
@
text
@d15 3
a17 2
Usage: debdeltas [ option...  ] deb_files
  Computes all missing deltas for Debian files deb_files
d24 1
d81 2
d104 1
a104 1
    ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
d117 5
d696 1
a696 1
        if VERBOSE >3 : print '   identical!  ', oldname, name, multiple
d701 1
a701 1
        if VERBOSE > 3 : print '   skip using old file ', name
d902 5
a906 3
    l= len( info_by_pack_arch[ (pa,ar) ] )

    if l > 1 :
d908 2
a909 3

      l -= 1
      while l>0:
d914 1
@


1.23
log
@Allow +- 10% on MAX_DELTA_PERCENT , depending on delta size.

debdeltas: '--avoid file' option, to avoid packages from a dist.

debdelta : really skip conf files !

Better verbosity yet.
@
text
@d474 2
a475 1
      print ' patching time: %dsec, speed:  %dkB per second ' % (a,(debsize / 1024 /  (a+.001)))
d661 11
d703 1
a703 5
          system("cat 'OLD/"+w+"/"+oldname+"' >> OLD/mega_cat")
          unlink('OLD/'+w+'/'+oldname)          
          script.write("cat 'OLD/"+w+"/"+oldname+"'  >> OLD/mega_cat\n")
          if not multiple:
            script.write("rm 'OLD/"+w+"/"+oldname+"'\n")
d715 2
a716 4
          system("cat 'OLD/"+w+"/"+oldname+"' >> OLD/mega_cat")
          unlink('OLD/'+w+'/'+oldname)
          script.write("cat 'OLD/"+w+"/"+oldname+"'  >> OLD/mega_cat ; rm 'OLD/"+w+"/"+oldname+"'\n")
    
a719 3
      if os.path.exists('OLD/mega_cat'):
        # if -k is given, still we need to delete it...
        os.unlink('OLD/mega_cat')
d1143 1
a1143 2
            if VERBOSE : print '   for ',name
            else: print 'Created ',newdeb
d1157 2
@


1.22
log
@debdelta' can use MD5 to exploit identical files that were renamed.
This can express the difference between tetex-doc 3.0-17 and 3.0-18
into  260kB , even though all the directory tree was moved around !

This debdelta is in Debian package version 0.8
@
text
@d76 1
d85 1
d100 2
a101 2
      ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
                                     ('help','info','needsold','dir=') )
d114 5
d163 15
a355 1
  s=p.readline()#skip #!/bin/sh
d357 3
d474 1
a474 1
      print ' patching time: %dsec, speed:  %dkB per second ' % (a,(debsize / 1024 /  (a+1)))
d579 1
a579 1
    old_conffiles=p.read().split('\n')
d832 2
a833 2
    print ' deb delta is  %3.1f%% of deb ; that is, %dkB would be saved' \
          % ( percent , (( newdebsize -deltasize ) / 1024) )
d836 1
a836 1
            (elaps, newdebsize / 1024 / (elaps+1))
d850 8
d901 8
a908 1

d911 2
a912 2
        if newdebsize <= 4 * 1024 :
          #this actually affects 1 every 60 packages in the archives
d921 1
a921 1
            a=DIR+'/'+os.path.dirname(new['File'])+'/'+deltabasename
d927 1
d936 1
a936 1
        if free and free < newdebsize /2 + 1024 :
d966 5
a970 1
          if ret[1] > MAX_DELTA_PERCENT and os.stat(delta)[ST_SIZE] >= 4*1024 :
d1069 1
a1069 1
def delta_upgrade(DIR):
d1080 2
d1086 31
a1116 5
    DIR='/tmp/archives'
  if not os.path.exists(DIR):    
    os.mkdir(DIR)

  print 'Recreated debs are saved in ',DIR
d1118 1
d1120 1
a1120 2
  len_deltas=0

d1141 1
a1156 3
        global len_newdebs
        if len_newdebs != None and os.path.exists(newdeb):
          len_newdebs += os.stat(newdeb)[ST_SIZE]
d1159 1
a1159 1
      return
d1164 3
d1170 6
a1175 15

      ## FIXME : how do I get the URI and/or architecture out of python-apt ??
      #if p.sourcePackageName[:3] == 'lib':
      #  b=p.sourcePackageName[:4]
      #else:
      #  b=p.sourcePackageName[0]
      #c=p.candidateOrigin[0].component
      ## rely on usual pool structure....
      #u=c+'/'+b+'/'+p.sourcePackageName
      ##this sucks... it is so slow..., but it works
      dpkg_params={}
      pip=os.popen('env -i dpkg -p '+p.name)
      scan_control(pip,dpkg_params)
      pip.close()
      arch=dpkg_params['Architecture']
d1177 1
a1177 3
      assert(dpkg_params['Version'] == p.installedVersion )


d1179 1
a1179 1
      if os.path.exists(DIR+'/'+newdeb) or \
d1188 1
a1188 1
      newdeb = DIR+'/'+newdeb
d1205 1
d1207 2
d1213 2
d1216 7
a1222 3
        j += 1
        sys.stderr.write(" %5d kB for %s \r" % ( j , delta_name))
        s=r.read(1024)        
d1225 1
d1232 5
d1244 6
a1249 3
  if VERBOSE:
    print 'Delta-upgrade download and patch time: %dsec; virtual speed: %dkB per second.' %  \
          (elaps, len_newdebs / 1024 / (elaps+1)) 
d1255 1
a1255 1
  delta_upgrade(DIR)
@


1.21
log
@delta_upgrade : is a function ;
 " : accepts --dir option
 " : patching queue ending was not managed OK

started preparing delta_tar in do_delta for md5 .
@
text
@d69 1
a69 1
from shutil  import rmtree
d134 6
d185 1
a185 1
  if a[-1] ==  '\n' :
d189 7
d507 37
a543 1
  
d639 2
a640 1
    oldnames = oldtar.getnames()
d644 5
a648 1
      name = newtarinfo.name
d650 2
a651 1
        oldname = renames[name]
d654 2
a655 1
      if  (('/'+name) in  skip ) or ( name in skip ):
d657 5
a661 2
      elif name in oldnames and  newtarinfo.isreg() :
        oldtarinfo = oldtar.getmember(newtarinfo.name)
d663 24
a686 7
          if VERBOSE > 4 : print '  use old file ', name
          oldtar.extract(oldtarinfo.name,"OLD/"+w )
          system("cat 'OLD/"+w+"/"+name+"' >> OLD/mega_cat")
          unlink('OLD/'+w+'/'+name)
          script.write("cat 'OLD/"+w+"/"+name+"'  >> OLD/mega_cat ; rm 'OLD/"+w+"/"+name+"'\n")
      elif VERBOSE > 4 : print '  not diffable from old : ', name

d690 3
d759 1
a759 1
      delta_tar(o,n,'DATA',old_conffiles)
d956 2
a957 1
          rmtree(T)
@


1.20
log
@debdelta-upgrade: use a separare thread (and fork :-( ) for the patching.

Again adjustments to VERBOSE... lets say that -vvv is a reasonable output.

debdelta-upgrade: use 'arch' from 'dpkg -p' when creating package file name.
@
text
@d112 1
a112 1
    elif o == '--dir'  and action == 'deltas' :
d124 2
a125 1

a177 4
#def symlink_w_parents(f,d):
#  d=make_parents(fd)
#  os.symlink(f,d)

a182 1

d585 2
a586 1
  def delta_tar(o,n,w,skip=()):
d595 4
d942 1
d945 1
a945 1
elif action == 'delta-upgrade' :
d967 1
d971 1
a971 1
  def thread_do_patch(qout,qin):
d982 1
a982 1
        # my fault... echo do_patch() wants its own CWD!
d1007 1
a1007 3
      if VERBOSE>=2 : print ' Patching thread loop ended. '
      s=os.write(qin,'\t')
      s=os.read(qout,2)
d1011 2
a1012 1
  thread_patch=thread.start_new_thread(thread_do_patch  ,(qout,qin))
d1083 2
a1084 2
  os.read(qout,1)
  #while thread_patch:    time.sleep(1)
d1092 4
d1097 1
a1097 1
elif  os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
@


1.19
log
@Allow -M in debdeltas.

Bug: do not remove non-existent file.
@
text
@d429 1
a429 1
    if VERBOSE > 2 : a = '-v'
d944 1
a945 4
  p=os.popen('dpkg --print-architecture')
  arch=de_n(p.read())
  p.close()
  
d950 1
a950 1
  
d966 46
a1015 12
      newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
      if os.path.exists(DIR+'/'+newdeb) or \
             os.path.exists('/var/cache/apt/archives/'+newdeb):
        if VERBOSE: print  'Already downloaded: ',newdeb
        continue

      if VERBOSE:
        print 'Looking for a delta for %s from %s to %s ' % \
              ( p.name, p.installedVersion, p.candidateVersion )

      newdeb = DIR+'/'+newdeb

d1029 2
a1030 2
      a=dpkg_params['Architecture']
      u=dpkg_params['Filename']
d1032 14
d1049 1
a1049 1
                  a+'.debdelta'
d1051 1
a1051 1
      uri=delta_http_base+'/'+os.path.dirname(u)+'/'+delta_name
d1073 11
a1083 11
      T=tempo()
      try:
        ret=do_patch(tmp_delta,'/',newdeb ,T)
        len_newdebs += os.stat(newdeb)[ST_SIZE]
      except DebDeltaError,s:
        print ' Error: applying of delta failed: ',str(s)
        if os.path.exists(newdeb):
          os.unlink(newdeb)
      unlink(tmp_delta)
      os.chdir(original_cwd)
      rmtree(T)
d1087 1
a1087 1
          (elaps, len_newdebs / 1024 / (elaps+1))
@


1.18
log
@On keyboard interrupt, clean up tmp files.

The test for xdelta VS bsdiff was not working (wrong parentheses?).
Moreover, my test shows that bsdiff uses 12 times the memory not 17.

Quote filenames in patch.sh.
@
text
@d110 1
a110 1
    elif o == '-M' and action == 'delta' :    MAXMEMORY = 1024 * 1024 * int(v)
d330 3
a332 1
      if VERBOSE > 1 or (VERBOSE and action != 'deltas' ) or INFO : print ' info: ',s
d1034 2
a1035 1
        os.unlink(newdeb)
@


1.17
log
@Recover upstream uri and architecture from 'dpkg -p' (very slow but it
works).

Adjustments to verbosity.

die() does not print anything; DebDeltaError ships the error message.
@
text
@d559 1
d561 1
d565 1
a565 1
    if osize < MAXMEMORY / 17 and osize * 8 < free  :
d569 1
a569 1
      if VERBOSE > 4 : print '  fallback on xdelta instead of bsdiff' 
d603 1
a603 1
          system('cat OLD/'+w+'/'+name+' >> OLD/mega_cat')
d605 1
a605 1
          script.write('cat OLD/'+w+'/'+name+'  >> OLD/mega_cat ; rm OLD/'+w+'/'+name+'\n')
d792 1
a792 1
          if VERBOSE :     print 'Skip , too small: ', new['File']
d824 4
d856 4
d867 1
d869 3
a871 2
            print " Unexpected error while testing delta:", sys.exc_info()[0]
            os.unlink(delta)
d899 3
d923 3
@


1.16
log
@Implements 'debdelta-upgrade' .

Start writing some kind of APT method

Change ':' to '%3a' in file names.
@
text
@d157 1
a157 1
    if a[:4] in ('Pack','Vers','Arch','Stat','Inst'):
d255 2
a256 2
  def __str__(self,s):
    return __str
d259 1
a259 1
  if s : sys.stderr.write(s+'\n')
d330 1
a330 1
      if VERBOSE or INFO : print ' info: ',s
d432 1
a432 1
    if VERBOSE > 1:
d726 1
a726 1
    if VERBOSE > 1:
d813 1
a813 1
          if VERBOSE : print ' Not enough disk space for',delta
d823 1
a823 1
        except DebDeltaError:
d827 2
a828 1
          print ' Creation of ',delta,' failed.'
d838 1
a838 1
          if ret[1] > MAX_DELTA_PERCENT:
d840 1
a840 1
            if VERBOSE : print ' Error, too big!'
d851 2
a852 2
          except DebDeltaError:
            print ' Error: testing of delta failed: ',delta
d881 2
a882 1
  except DebDeltaError:
d902 2
a903 1
  except DebDeltaError:
d917 2
a918 1
  except DebDeltaError:
d932 1
a932 1
  delta_http_base='/mirror/debian-deltas/pool/'
d946 3
a948 1
  
d964 17
a980 1
      
d983 5
a987 9
                  '_'+ version_mangle(p.candidateVersion)+'_i386.debdelta'
      ## FIXME : how do I get the URI out of python-apt ??
      if p.sourcePackageName[:3] == 'lib':
        b=p.sourcePackageName[:4]
      else:
        b=p.sourcePackageName[0]
      c=p.candidateOrigin[0].component
      ## rely on usual pool structure....
      uri=delta_http_base+c+'/'+b+'/'+p.sourcePackageName +'/'+delta_name
d1011 3
a1013 2
      except DebDeltaError:
        print ' Error: applying of delta failed: ',delta
d1018 4
a1021 2

  
@


1.15
log
@Catch getopt exception.

1/60th of packages is of size less than 4kB ; and those produce "large" deltas.
@
text
@d15 2
a16 2
Usage: debdeltas [ option...  ] debs
  Computes all missing deltas for debs
d21 2
a22 2
            if DIR ends in // , then the dirname of the arguments will be used as well
--search    search in the directory of the above debs for older versions
d27 2
d42 9
a73 16

action=(os.path.basename(sys.argv[0]))[3:]
actions =  ('delta','patch','deltas')
if action not in actions:
  print 'wrong filename: should be "deb" + '+repr(actions)
  sys.exit(0)

__doc__ = doc[action] + doc_common

try: 
  ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
                                 ('help','info','needsold','dir=') )
except getopt.GetoptError,a:
  sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n')
  sys.exit(2)

d85 8
a92 14
for  o , v  in  opts :
  if o == '-v' : VERBOSE += 1
  elif o == '-d' : DEBUG += 1
  elif o == '-k' : KEEP = True
  elif o == '--needsold' and action == 'delta' :  NEEDSOLD = True
  elif o == '-M' and action == 'delta' :    MAXMEMORY = 1024 * 1024 * int(v)
  elif o == '--info' and action == 'patch' : INFO = True
  elif o == '--dir'  and action == 'deltas' :
    DIR = v
    if not os.path.isdir(DIR):
      print 'Error: --dir ',DIR,' does not exist.'
      sys.exit(3)
  elif o ==  '--help' or o ==  '-h':
    print __doc__
a93 3
  else:
    print ' option ',o,'is unknown, try --help'
    sys.exit(1)
d95 29
a123 9
if INFO  :
  if  len(argv) > 1 and VERBOSE :
    print '(printing info - extra arguments are ignored)'
  elif  len(argv) == 0  :
    print ' need a  filename ;  try --help'
    sys.exit(1)
elif action != 'deltas' and len(argv) != 3 :  
  print ' need 3 filenames ;  try --help'
  sys.exit(1)
d234 1
d236 5
d794 2
a795 1
        deltabasename = pa +'_'+  old['Version'] +'_'+ new['Version'] +'_'+ar+'.debdelta'
d839 1
a839 1
            if VERBOSE : print ' Error, too big:',delta
d866 10
d891 5
a895 1
elif action == 'delta' :  
d917 89
d1007 99
a1105 1
    
@


1.14
log
@Corrected some bugs.

Some checks for disk space.
@
text
@d72 6
a77 4
( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
                               ('help','info','needsold','dir=') )


d775 3
a777 1
        if newdebsize <= 20 * 1024 :
@


1.13
log
@'debdeltas' to scan archive and create many deltas

'debdelta' deals with dpkg diversions

main operations are now functions

errors are reported using exceptions

reviewed verbosity
@
text
@d78 2
a79 1
MAX_DELTA_PERCENT = 40
d95 5
a99 1
  elif o == '--dir'    and action == 'deltas' : DIR = v
a125 2


d149 1
a149 1
    if a[:3] in ('Pac','Ver','Arc','Sta'):
d160 1
a160 1
  d='/'
a237 1

d239 4
a242 1
  pass
d325 6
d357 1
a357 1
        if a[:3] == 'OLD':
d382 1
d388 1
a388 1
            s.append(orig,divert)
d393 2
a394 2
        for orig,divert in s:
          if os.path.isfile(divert) and not os.path.islink(divert) :
d396 1
d398 2
d482 7
d740 1
a740 1
      info_by_file[f]['File'] = abspath(f)
d772 3
a774 3

        if os.stat(new['File'])[ST_SIZE] < 16 * 1024 :
          if VERBOSE > 2:     print 'Skip , too small: ', new['File']
d781 2
a782 2
            a=os.path.dirname(new['File'])+'/'+deltabasename
            delta=make_parents(a,DIR)
a793 1

d795 1
a795 1
        if free and free < 2 ** 20 :
d804 3
d808 3
a810 3
        except DebDeltaError:
          os.unlink(delta)
          print 'Creation of ',delta,' failed.'
d812 3
a814 1
          os.unlink(delta)
d816 2
a817 1
          rmtree(T)
d822 1
a822 1
            if VERBOSE : print 'Error, too big:',delta
d831 4
d836 2
d839 2
a840 2
            print "Unexpected error:", sys.exc_info()[0]
            print 'Error: applying of delta failed: ',delta
d843 1
a844 1
        
a845 2

  
d854 2
a855 1
    os.unlink(newdeb)
d859 2
a860 1
    os.unlink(newdeb)
d871 2
a872 1
    os.unlink(delta)
d875 2
a876 1
    os.unlink(delta)
@


1.12
log
@Typo.
@
text
@d3 2
a4 1
"""\
d8 20
d29 1
a29 1
  Applies patchin to fromfile and produces  a  reconstructed  version of tofile.
d37 2
a38 6
Options for debdelta:
--needsold  create a patch that can only be used if the old .deb is available
  -M Mb   maximum memory (to decide if using 'bsdiff' or 'xdelta')

Options for debpatch:
 --info  print info on two Debian files, and exists
d40 1
a40 1
Options for both:
d60 1
a60 1
####################################################################
a61 1
start_sec = time.time()
d65 1
a65 1
actions =  ('delta','patch')
d70 2
d73 2
a74 1
                               ('help','info','needsold') )
a75 1
original_cwd = os.getcwd()
d78 2
d85 1
d94 1
d108 1
a108 1
elif len(argv) != 3  or ( len(argv) != 3   ):  
d112 21
a132 1
######################################################################
d138 5
a142 1
def scan_control(p,params,prefix,script=None,stdout = None):
d146 2
a147 2
    if a[:3] in ('Pac','Ver','Arc'):
      if script : script.write('#'+prefix+'/'+a+'\n')
d151 1
a151 1
      params[prefix+'/'+a[:i]] = a[i+2:]
d154 4
a157 2
def symlink_w_parents(f,d):
  s=f.split('/') 
d160 1
a160 1
      d = d + '/' + a
d162 7
a168 4
        os.mkdir(d)      
  d += '/'+s[-1]
  os.symlink(f,d)

d201 1
a201 1
def unpack(d,f):
d205 1
a205 1
  os.chdir(TD+'/'+d)
a222 8
####################################################################
if KEEP:
  def unlink(a):
    if VERBOSE : print ' would unlink ',a
  def rmdir(a):
    if VERBOSE : print ' would rmdir ',a
  def rmtree(a):
    if VERBOSE : print ' would rm -r ',a
a223 4
TD = abspath(tempfile.mkdtemp())
for i in 'OLD','NEW','PATCH' :
  os.mkdir(TD+'/'+i)
if  VERBOSE > 1 or KEEP :  print 'temporary in '+TD
d225 6
d234 5
d241 1
a241 2
  rmtree(TD)
  sys.exit(2)
d248 2
d271 9
a279 2
if action == 'patch':  
  delta = abspath(argv[0])
a285 1
    olddeb = abspath(argv[1])
d288 1
a288 3

    newdeb = abspath(argv[2])
    if  os.path.exists(newdeb) :
d290 1
a290 2
    
    unpack ('PATCH',delta)
d334 1
a334 1
        p=os.popen('dpkg -s '+b)
d339 6
d358 1
a358 1
      unpack ('OLD',olddeb)
d364 1
d366 19
a384 11
        a=params['OLD/Package']
        b='/var/lib/dpkg/info/' + a +'.list'
        if not os.path.exists(b ):
          die('Package "'+a+'" is not installed ??')
        p=open(b)
        s=p.read().split('\n')
        p.close()
        os.mkdir(TD+'/OLD/DATA')
        for a in s:
          if os.path.isfile(a) and not os.path.islink(a) :
            symlink_w_parents(a, TD+'/OLD/DATA')
a385 1
        os.mkdir(TD+'/OLD/DATA')
d400 2
a401 1
    shutil.move('NEW.file',newdeb)
d404 4
a407 1
      newdebsize = os.stat(newdeb)[ST_SIZE]
d410 1
a410 1
      print ' time: %dsec, speed:  %dkB per second ' % (a,(newdebsize / 1024 /  (a+1)))
d413 5
a417 2
elif action == 'delta' :
  olddeb = abspath(argv[0])
d420 2
a421 2
  
  newdeb = abspath(argv[1])
d425 6
a430 2
  
  delta = abspath(argv[2])
d483 11
a493 1
    
d524 4
a527 7
    try:
      a=os.statvfs(TD)
      freespace= a[0] * a[4]
    except a:
      if VERBOSE : print ' statvfs error ',a
      freespace = MAXMEMORY * 16
    if osize < MAXMEMORY / 17 and osize * 8 < freespace   :
d531 1
a531 1
      if VERBOSE > 2 : print '  fallback on xdelta instead of bsdiff' 
d539 1
a539 1
    if VERBOSE > 1 :
d554 1
a554 1
    
d563 1
d568 8
a575 2
    delta_files('OLD/mega_cat',n)
    #clean up
a576 1
    rmtree('OLD/'+w)
d604 1
a604 1
    if VERBOSE > 1: print ' studying ' , name , ' of len ' , newsize
d607 1
a607 1
    if VERBOSE > 2: print '  ar line: ',repr(s)
d642 2
a643 7
      deltacount += 1
      pp=str(deltacount)
      p = 'PATCH/'+pp
      if VERBOSE > 3 : print '   including "',name,'" verbatim in patch'
      os.rename(n,p)
      patch_append(pp)
      script.write('cat '+p+' >> NEW.file ; rm '+p+'\n')      
d684 3
a686 1

d688 2
a689 3
    print ' deb delta is  %3.1f'  % \
          ( deltasize * 100. /  newdebsize ) ,    '% of deb'
    print '  that is, %dkB would be saved ' % (( newdebsize -deltasize ) / 1024)
d691 48
a738 3
      end_sec = time.time()
      a=(end_sec - start_sec)
      print '  time: %dsec, speed: %dkB per second ' % (a,newdebsize / 1024 / (a+1))
d740 2
a741 4
####################################
else:
  #unimplemented action
  assert(0)
d743 69
a811 3
#cleanup
os.chdir(original_cwd)
rmtree(TD)
d814 2
a815 3
##   a='ar qSc result.deb '
##   for o in arlist['NEW'] :
##     a=a+ ' ' + o + ' '
d817 13
a829 3
##     '!<arch>\n'

##   a='fakeroot sh -c "chown root.root * ; ' + a + ' " '
d831 13
a843 1
##   s.write()
d845 5
a849 1
##   S(' cd NEW ; ' + a)
a850 47
##   ret=os.system('cmp NEW/result.deb '+newdeb )

##   if ret:
##     S('xdelta delta -n -9 '+newdeb+' NEW/result.deb ')
##   #S(['ar','qSc', 'temp.deb',]+ deltaparts)
##   #s.write('xdelta patch '+o+'.xdelta'+' ../OLD/'+o+' '+o+'\n')
  
##   def a(p,k,v):
##     if  p == None:
##       p = {}
##     if len(k) > 1  :
##       p[k[0]] = a(p.get(k[0]) , k[1:] ,v  )
##     else:
##       p[k[0]] = v
##       return p    

##     if '/' in s:
##       s=s.split('/')
##       if (len (s) == 2) :
##         ( a,  v ) = s
##         if '/' in s:
##           ....
##         params[ a  ] = v

##       elif (len (s) == 3) :
##         ( a, b, v ) = s
##         if a not in params :
##           params[a] ={}
##         params[ a ][b] = v
##       else:
##         print 'internal error on parm ', repr(s)


##     if False and DEBUG:
##       a=params['OLD/Package']
##       b='/var/lib/dpkg/info/' + a +'.list'
##       if os.path.exists(b ):
##         p=open(b)
##         s=p.read()
##         s=s.split('\n')
##         p.close()
##         for b in s :
##           if not ( b[1:] in oldnames ) :
##             print ' CASINO ',b
##         for b in oldnames : assert( '/'+b in s )
##       else:
##         print ' (package is not installed )',b
d852 1
@


1.11
log
@Added yet another die()
@
text
@d204 1
a204 1
    die('Error: '+f + ' does not exists.')
d212 1
a212 1
    die('Error: '+f + ' does not exists ')
@


1.10
log
@If -vv , display time to compute, and kB per second.

Use die() when files are not debs , or are not existant.
@
text
@d212 1
a212 1
    print f , ' does not exists '
@


1.9
log
@Option '--fs' is no more; new option '--needsold' (that is the opposite).

When error, invoke 'die()' that prints error and cleans up.

Treat 'control.tar.gz' as we treat 'data.tar.gz'.

In patch.sh, call './minigzip' and not 'minigzip'.
@
text
@d36 1
a36 1
import sys , os , tempfile , string ,getopt , tarfile , shutil
d45 3
a117 17
def check_deb(f):
  if not  os.path.isfile(f) :
    print f , ' does not exists '
  p=open(f)
  if p.read(21) != "!<arch>\ndebian-binary" :
    print f , ' does not seem to be a Debian package '
    sys.exit(1)
  p.close()

def check_diff(f):
  if not  os.path.isfile(f) :
    print f , ' does not exists '
  p=open(f)
  if p.read(8) != "!<arch>\n" :
    print f , ' does not seem to be a Debian delta '
    sys.exit(1)
  p.close()
d202 16
d227 5
a231 1
  else:
a235 4
    olddeb = abspath(argv[1])
    if olddeb != '/':
      check_deb(olddeb)
    
d333 6
a338 1
    
d601 6
a606 1
    print ' that is, %dkB would be saved ' % (( newdebsize -deltasize  ) / 1024)
@


1.8
log
@Can create deltas that can be used to recreate a new .deb using the
the installed of the old .deb.

--info is now only a 'debpatch' option ;
 debdelta always include the info.
@
text
@d10 3
d17 1
a17 3
  --fs    create a patch that can be used to recreate the new .deb
          from the old deb that is installed in the host.
          In this case, when using 'debpatch', use '/' for fromfile.
d19 1
d22 1
a23 1
  -d      debug : add md5sums, check installed version for --fs
d28 3
d41 1
d52 1
a52 1
                               ('help','info','fs') )
d54 1
a54 1
cwd = os.getcwd()
d61 1
a61 1
FS      = False
d67 1
a67 1
  elif o == '--fs' and action == 'delta' : FS = True
a114 8
def system(a):
  if type(a) != type('') :
    a=string.join(a,' ')
  ret = os.system(a)
  if  ret != 0 and ( ret != 256 or a[:6] != 'xdelta') :
    print ' error , non zero return status ',ret,' for ',a
    sys.exit(2)

d192 2
a193 1

d198 4
a201 1
if  VERBOSE > 1 :  print 'temporary in '+TD
d203 4
d209 7
d245 1
a245 1
  #lets see what it does and what it requires
d247 2
d265 1
d267 3
a269 1
    if olddeb != '/':
d271 4
a274 2

    if olddeb == '/' and DEBUG:
d276 5
a280 1
      p=os.popen('dpkg -s '+params['OLD/Package'])      
d283 10
a292 6
      for a in  dpkg_params:
        if  params[a] != dpkg_params[a] :
          print 'Error : in installed version , '+a+' = ' +dpkg_params[a]
          print '         in debdelta version , '+a+' = ' +params[a]          
          sys.exit(2)
      
d295 1
a295 1
        raise 'needs old version Debian package'
d299 1
a299 1
        raise 'needs old version Debian package'
d306 1
a306 1
          raise ' package "'+a+'" is not installed ??'
d318 9
d330 2
a331 13

    if 'old-data-tree' in params :
      shutil.rmtree('OLD/DATA')

    os.rename('NEW.file',newdeb)

  for o in  'PATCH/patch.sh','PATCH.file','minigzip','OLD.file':
    if os.path.exists(o):
      if VERBOSE > 5 : print ' deleting ',o
      unlink(o)
  for o in os.listdir('OLD'):
    if VERBOSE > 5 : print ' deleting OLD/',o
    unlink('OLD/'+o)
d412 1
a412 1
      script.write('minigzip -9 '+n+'\n')
d456 1
a456 1
  def delta_data(o,n):
d465 3
a467 3
      if  ('/'+name) in  old_conffiles :
        if VERBOSE > 3 : print '   skip conffile ', name
      elif name in oldnames and  newtarinfo.isreg()  :
d470 5
a474 5
          oldtar.extract(oldtarinfo.name,"OLD/DATA" )
          system('cat OLD/DATA/'+name+' >> OLD/data_mega_cat')
          unlink('OLD/DATA/'+name)
          script.write('cat OLD/DATA/'+name+'  >> OLD/data_mega_cat ; rm OLD/DATA/'+name+'\n')
    delta_files('OLD/data_mega_cat',n)
d477 1
a477 1
    shutil.rmtree('OLD/DATA')
d482 1
a482 1
  if not  FS:
d523 2
a524 2
    elif False and name[:11] == 'control.tar' :
      #TODO
d526 11
a536 1
    elif FS and name[:8] == 'data.tar'  :
d540 1
a540 1
      delta_data(o,n)
d542 1
a542 1
    elif  FS or name not in ar_list_old :       #or it is not in old deb
d550 1
a550 1
    elif not FS:
d558 1
a558 1
      raise
a588 3
  shutil.rmtree('OLD/CONTROL')
  shutil.rmtree('NEW/CONTROL')
    
a589 2
  unlink('NEW.file')
  unlink('OLD.file')
d601 2
a602 5
os.chdir(TD)
rmdir('PATCH')
rmdir('OLD')
rmdir('NEW')
rmdir(TD)
d664 2
@


1.7
log
@'debpatch --info' accepts only 1 argument.

When old ar component was missing, must 'patch_append(p)' it.
@
text
@d14 1
a14 1
  --fs    create a (larger) patch that can be used to recreate the new .deb
d17 1
a17 3
  -d      debug : add md5sums to patch
 --noinfo do not insert in patch the info on two Debian files
  -M Mb   maximum memory (decides between using 'bsdiff' or 'xdelta')
d21 2
a22 1
  -v      verbose (can be added multiple times
d31 1
a31 1
import sys , os , tempfile , string ,getopt
d51 1
a51 1
DEBUG   = 0
d54 1
a54 1
INFO    = action == 'delta'
d59 1
a59 1
  elif o == '-d' and action == 'delta' : DEBUG += 1
d63 1
a63 2
  elif o == '--info' : INFO = True
  elif o == '--noinfo' : INFO = False 
d71 1
a71 1
if INFO and action == 'patch' :
d82 27
d153 12
d199 1
a199 1
if DEBUG or VERBOSE > 1 :  print 'temporary in '+TD
d237 9
a245 8
  while s and s[0] == '#':
    s=de_n(s)[1:]
    if VERBOSE or INFO : print ' info: ',s
    if ':' in s:
      i=s.index(':')  
      params[s[:i]] = s[i+1:]
    else:
      params[s] = True
d251 14
d269 5
a273 3
      os.symlink(olddeb,TD+'/OLD.file')
    
    if 'needs-old' in params:
d275 14
a288 2
        raise 'needs old version Debian package'
      os.symlink(olddeb,TD+'/OLD.file')
d294 3
a332 15
  ##### write parameters
  if DEBUG or INFO:
    for o in 'OLD', 'NEW' :
      if INFO : print o
      system('ar p  '+o+'.file control.tar.gz | tar xzf - ./control')
      p=open('control')
      a=p.readline()
      while a:
        a=de_n(a)
        if a[:3] in ('Pac','Ver','Arc'):
          if DEBUG : script.write('#'+o+'/'+a+'\n')
          if INFO : print ' ' , a
        a=p.readline()
      p.close()
    unlink('control')
d335 1
d339 16
d356 8
a363 5
  #this delta needs the old deb , unpacked in 'OLD'
  #script.write('#unpack-old\n')
  #this delta needs the old deb 
  script.write('#needs-old\n')

d372 1
a372 1
  def unzip(f):
d376 1
a376 1
      if f[:3] != 'NEW' :
a421 1

d430 30
a459 1
  ############# start scanning the new deb
d463 1
a470 1

d497 10
a506 1
    elif  name not in ar_list_old :       #or it is not in old deb
d508 3
a510 1
      p = 'PATCH/'+str(deltacount)
d512 3
a514 4
      patch_append(p)
      script.write('echo PATCH/'+o+' >> NEW.file')
      if DEBUG: script.write('rm PATCH/'+o+'\n')
    else:
d521 2
d543 12
a554 4
  script.close()  
  system('gzip -9 -n PATCH/patch.sh')
  
  patch_append('patch.sh.gz')
d620 16
@


1.6
log
@Use bsdiff when memory does not exceed 50Mb, and free disk space is enough;

' debpatch --info  patch' to just know info on a patch

Shipped in packagde 0.3
@
text
@d14 1
a14 1
  --fs    TODO: create a (larger) patch that can be used to recreate the new .deb
d73 7
a79 1
if len(argv) != 3  :  
d81 1
a81 1
  sys.exit(0)
d232 1
d235 1
a235 1
    if DEBUG: print ' deleting OLD/',o
a252 3
  #unpack('OLD',olddeb)
  #unpack('NEW',newdeb)

a255 2
  #components of this patch
  deltaparts=['patch.sh.gz']
a326 1
    #deltaparts.append(str(deltacount))
d399 1
a399 1
      deltaparts.append(p)
@


1.5
log
@This version uses bspatch / bsdiff ... but it uses too much memory.
@
text
@d3 2
a4 1
"""debdelta [ option...  ] fromfile tofile patchout
d7 1
a7 1
debpatch [ option...  ] patchin  fromfile  tofile 
d10 5
a14 3
Options for debpatch:
  --fs    TODO
          create a (larger) patch that can be used to recreate the new .deb
d18 4
d23 1
a23 1
  -v      verbose
a24 1
  --info  print info on two Debian files
d46 1
a46 1
( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhd' ,
d51 1
d53 1
a53 1
VERBOSE = 1
d55 1
a55 1
INFO    = False
d62 4
a65 2
  elif o == '--fs' and action == 'delta' : FS = True  
  elif o == '--info' : INFO = True  
a72 3

#should use getopt.gnu_getopt

d87 2
d96 2
d109 2
a110 2
def unpack(d,f):
  "unpacks 'ar' file f in directory d"
d112 1
a112 4

  os.symlink(f,TD+'/'+d+'.file')
  
  arlist[d] = []
d118 1
a118 1
    arlist[d].append(a)    
d120 2
d123 3
d158 1
a158 1
arlist = {}
d162 1
a162 5
if action == 'patch':
  newdeb = abspath(argv[2])
  if  os.path.exists(newdeb) :
    os.rename(newdeb,newdeb+'~')
  
a164 5
  
  olddeb = abspath(argv[1])
  check_deb(olddeb)

  unpack ('PATCH',delta)
d166 13
d181 1
a181 1

d184 1
a184 1
  if 'patch.sh.gz' in  arlist['PATCH']:
d186 2
a187 2
  elif 'patch.sh.bz2' in  arlist['PATCH']:
    system('bunzip2 PATCH/patch.sh.bz2')
d192 1
a192 1
  s=p.readline()
d195 2
a196 1
    s=de_n(s)[1:]    
d198 2
a199 3
      (a , b) = s.split(':')
      b = b[1:] 
      params[a] = b      
a204 1
  if VERBOSE : print ' info:',repr(params)
d206 15
a220 4
  if 'unpack-old' in params:
    unpack ('OLD',olddeb)
  
  system('/bin/sh -e PATCH/patch.sh')
d222 1
a222 1
  os.rename('NEW.file',newdeb)
d235 2
a236 1

d239 1
d246 2
a247 2
  unpack('OLD',olddeb)
  unpack('NEW',newdeb)
d255 1
d266 1
a266 1
      system('tar xzf '+o+'/control.tar.gz ./control')
d272 1
a272 1
          if DEBUG : script.write('#'+o+':'+a+'\n')
d284 72
a355 2
  script.write('#unpack-old\n')
  
d357 3
a359 1
    
d367 10
a376 4
    
  for o in arlist['NEW'] :
    oldsize = os.stat('NEW/'+o)[ST_SIZE]
    if VERBOSE > 1: print 'studying ',o,' of len ',oldsize
d379 2
a380 2
    if VERBOSE > 2: print 'ar line: ',repr(s)
    assert( s[:len(o)] == o and s[-2] == '`' and s[-1] == '\n' )
d383 2
a384 2
    newdeb_file.seek(oldsize  ,1)
    if oldsize & 1 :
d389 3
a391 3
    if oldsize < 128:      #file is too short to compute a delta,
      p=open('NEW/'+o)
      append( p.read(oldsize))
d393 6
a398 6
      unlink('NEW/'+o)
      if o in arlist['OLD'] :
        unlink('OLD/'+o)
    elif  o not in arlist['OLD'] :       #or it is not in old deb
      os.rename('NEW/'+o,'PATCH/'+o)
      deltaparts.append(o)
d403 6
a408 30
      c=''
      if o[-3:] == '.gz' :
        #cannot gunzip if there is a link ! os.link('NEW/'+o,'tmp_n')
        o=o[:-3]
        system('gunzip  NEW/'+o+'.gz')
        system('gunzip  OLD/'+o+'.gz')
        c='.gz'
      elif  o[-3:] == '.bz2' :
        print 'WARNING ! ',o,' is in BZIP2 format ! please fixme !'
      system('bsdiff  OLD/'+o+' NEW/'+o+' PATCH/'+o+'.bsdiff')
      deltaparts.append(o+'.bsdiff')
      unlink('NEW/'+o)
      unlink('OLD/'+o)
      ## how did we fare ?
      deltasize = os.stat('PATCH/'+o+'.bsdiff')[ST_SIZE]
      if VERBOSE > 1 :
        print ' delta is  %3.4f'  % ( deltasize * 100. /  oldsize ) , '% of ',o
      elif  (deltasize > oldsize  and DEBUG): 
        print 'this sucks: deltasize ',deltasize,' > oldsize ',oldsize
      ## and prepare the script consequently
      if c == '.gz':
        script.write('gunzip OLD/'+o+'.gz\n')  
      script.write('bspatch OLD/'+o+' NEW/'+o+' PATCH/'+o+'.bsdiff\n')
      script.write('rm PATCH/'+o+'.bsdiff OLD/'+o+'\n')
      if c == '.gz' :
        script.write('minigzip -9 NEW/'+o+'\n')
      if DEBUG:
        pass #implement MD5
      script.write('cat NEW/'+o+c+' >> NEW.file\n')
      script.write('rm NEW/'+o+c+'\n')
a412 1
  if VERBOSE > 2: print ' ar leftover character: ',repr(s)
d414 1
d418 8
a425 1
    pass #implement MD5
d430 2
a431 7
  #create final debdelta
  os.chdir(TD+'/PATCH')  
  system(['ar','qSc', delta,]+ deltaparts)
  for o in deltaparts:
    unlink(o)

  os.chdir(TD)
d473 24
@


1.4
log
@Added getopt support.
Reorganized code.
Be careful of cwd when using os.path.abspath.
Add parameters to patch.sh , in particular,
 support for '#unpack-old' keyword
Graduated verbosity.
@
text
@d144 1
a144 1
if DEBUG :  print 'temporary in '+TD
d177 1
d181 3
a183 2
      i=s.index(':')
      params[s[:i]] = s[i+1:]
d226 2
d305 2
a306 4
      s= '-n'
      if DEBUG: s=''
      system('xdelta delta '+s+' -9 OLD/'+o+' NEW/'+o+' PATCH/'+o+'.xdelta')
      deltaparts.append(o+'.xdelta')
d310 1
a310 1
      deltasize = os.stat('PATCH/'+o+'.xdelta')[ST_SIZE]
d318 2
a319 2
      script.write('xdelta patch PATCH/'+o+'.xdelta OLD/'+o+' NEW/'+o+'\n')
      script.write('rm PATCH/'+o+'.xdelta OLD/'+o+'\n')
@


1.3
log
@can build diff of two .debs and patch one to get the other
(debian version 0.1)
@
text
@d3 2
a4 3
__doc__ = """
   debelta
       The debdelta command has the following synopsis:
d6 2
a7 1
       debdelta [ option...  ] fromfile tofile patchout
d9 10
a18 9
       Computes a delta from fromfile to tofile and writes it to patchout

   debpatch
       The debpatch command has the following synopsis:

       debpatch [ option...  ] patchin  fromfile  tofile 

       Applies patchin to fromfile and produces  a  reconstructed  version  of
       tofile.
a22 5
DEBUG   = True
VERBOSE = 1
KEEP    = False 

actions =  ('delta','patch')
d26 1
a26 1
import sys , os , tempfile , string
d34 38
d113 2
a114 1
  
d117 1
a117 1
  os.chdir(TD)
d120 1
a120 1
ALLOWED = '<>()[]{}.,;:_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
a139 12
#should use getopt.gnu_getopt

if len(sys.argv) <= 1 or sys.argv[1] == '--help' or sys.argv[1] == '-h' :
  print __doc__
  sys.exit(0)

action=(os.path.basename(sys.argv[0]))[3:]

if action not in actions:
  print 'wrong action: may be '+repr(actions)

argv=sys.argv[1:]
a156 2

  unpack ('PATCH',delta)
d161 2
d230 1
a230 1
  if DEBUG:
d232 1
d239 2
a240 1
          script.write('#'+o+a+'\n')
a244 4
  #this delta needs the old deb , unpacked in 'OLD'
  script.write('#unpack-old\n')

  
d249 4
d265 1
a265 1
    if VERBOSE: print 'studying ',o,' of len ',oldsize
d268 1
a268 1
    if VERBOSE > 1: print 'ar line: ',repr(s)
d309 1
a309 1
      if VERBOSE :
d328 1
a328 1
  if VERBOSE: print 'leftover: ',repr(s)
d351 1
a351 1
    print ' deb delta is  %3.4f'  % \
d353 1
@


1.2
log
@this works, it creates deltas that can rebuild the exact .deb
@
text
@d20 1
a20 1
minigzip='/home/andrea/bin/minigzip'
d60 26
a85 1
ALLOWED = '. abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
d111 1
a111 1
action=sys.argv[0][-5:]
d118 4
d123 1
a123 17
if action == 'delta' :
  olddeb = abspath(argv[0])
  check_deb(olddeb)

  newdeb = abspath(argv[1])
  check_deb(newdeb)
  newdebsize = os.stat(newdeb)[ST_SIZE]
  
  delta = abspath(argv[2])
  if  os.path.exists(delta) :
    os.rename(delta,delta+'~')
  
  pairs = [ ('OLD',olddeb) , ('NEW',newdeb) ]
  
elif action == 'patch':
  olddeb = abspath(argv[1])
  check_deb(olddeb)
d125 1
d127 1
a133 3
  pairs = [ ('OLD',olddeb) , ('PATCH',delta) ]
else:
  assert(0)
d135 4
a138 4
TD = abspath(tempfile.mkdtemp())
if DEBUG :  print 'temporary in '+TD
#from here on, we live in the temp dir
os.chdir(TD)
d140 2
a141 1
arlist = {}
d143 1
a143 3
##################
for (d,f) in pairs :
  assert(os.path.exists(f))
d145 17
a161 10
  #os.symlink(f,TD+'/'+d+'.file')
  
  arlist[d] = []
  p=os.popen('ar t '+f,'r')
  while 1:
    a=p.readline()
    if not a : break
    if a[-1] ==  '\n' :
      a = a[:-1]
    arlist[d].append(a)    
d164 1
a164 6
  os.mkdir(d)  
  os.chdir(d)
  system('ar xo '+f)
  os.chdir(TD)
  
################# compute patch
d166 2
a167 4
if action == 'patch':
  os.symlink(minigzip,'minigzip')
  
  os.mkdir(TD+'/NEW')
a168 1
  system('gunzip PATCH/patch.sh.gz')
d173 8
a180 1
############## compute delta
d182 17
a198 1
  os.mkdir(TD+'/PATCH')
d206 18
d228 2
d264 1
d314 1
a314 1
  os.chdir(TD+'/PATCH')
d318 1
d320 1
a320 5
  rmdir('PATCH')
  rmdir('OLD')
  rmdir('NEW')
  rmdir(TD)
  
d322 10
a331 3
  
  print ' deb delta is  %3.4f'  % ( deltasize * 100. /  newdebsize ) , '% of deb'
  
d333 6
@


1.1
log
@Initial revision
@
text
@d4 2
a5 2
   Delta
       The delta subcommand has the following synopsis:
d7 1
a7 1
       debdelta delta [ option...  ] fromfile tofile patchout
d11 2
a12 2
   Patch
       The patch subcommand has the following synopsis:
d14 1
a14 1
       debdelta patch [ option...  ] patchin [ fromfile [ tofile ]]
d20 6
a25 1
minigzip='~/bin/minigzip'
d28 3
a30 1
import sys,os,tempfile , string
d36 1
a36 1
DEBUG =True
d38 41
a78 4
#def unlink(a):
#  print ' unlink ',a
#def rmdir(a):
#  print ' rmdir ',a
d82 1
a82 1
if len(sys.argv) <= 1 or sys.argv[1] == '--help' :
d86 1
a86 1
action=sys.argv[1]
d91 1
a91 1
argv=sys.argv[2:]
a92 3
TD = tempfile.mkdtemp()

if DEBUG :  print 'temporary in '+TD
d96 2
d99 1
d101 1
a105 1
  os.mkdir(TD+'/PATCH')
d107 1
d110 3
d114 5
a118 1
  delta = abspath(argv[0]) 
d123 3
a125 8
def S(a):
  if type(a) != type('') :
    a=string.join(a,' ')
  ret = os.system(a)
  if  ret != 0 and ( ret != 256 or a[:6] != 'xdelta') :
    print ' error , non zero return status ',ret,' for ',a
    sys.exit(2)
  
d129 2
a130 1
  
a131 3
  os.chdir(TD)
  os.mkdir(d)
  os.chdir(d)
d133 3
d143 1
a143 2
    arlist[d].append(a)
    
d145 4
a148 1
  S('ar xo '+f)
d151 1
a151 1
os.chdir(TD)
d154 8
a161 4
  os.chdir(TD+'/PATCH')
  S('gunzip patch.sh.gz')
  S('/bin/sh patch.sh')
  os.rename('result.deb',newdeb)
d163 1
d165 1
d168 1
d170 12
a181 2
  s=open('PATCH/patch.sh','w')
  s.write('#!/bin/sh -e\n')
d184 1
a184 1
  
d187 21
a207 1
    if o not in arlist['OLD'] or oldsize < 128:
a208 1
      unlink('OLD/'+o)
d210 1
d212 1
d217 2
a218 2
        S('gunzip -cv NEW/'+o+'.gz > ' + 'NEW/'+o)
        S('gunzip -cv OLD/'+o+'.gz > ' + 'OLD/'+o)
d220 6
a225 14
      S('xdelta delta -n -V -9 OLD/'+o+' NEW/'+o+' PATCH/'+o+'.xdelta')      
      deltasize = os.stat('PATCH/'+o+'.xdelta')[ST_SIZE]
      if deltasize > oldsize  :
        print 'bello schifo ',deltasize,' > ',oldsize
      if 1:
        if c == '.gz':
          s.write('gunzip ../OLD/'+o+'.gz\n')  
        deltaparts.append(o+'.xdelta')
        s.write('xdelta patch '+o+'.xdelta'+' ../OLD/'+o+' '+o+'\n')
        s.write('rm '+o+'.xdelta ../OLD/'+o+'\n')
        if c == '.gz' :
          s.write(minigzip+' -9 '+o+'\n')
          if DEBUG:
            pass
d228 32
a259 23
      if c:
        unlink('NEW/'+o+c)
        unlink('OLD/'+o+c)

  a='ar qSc result.deb '
  for o in arlist['NEW'] :
    a=a+ ' ' + o + ' '

  a='fakeroot sh -c "chown root.root * ; ' + a + ' " '
  
  s.write()
  
  S(' cd NEW ; ' + a)

  ret=os.system('cmp NEW/result.deb '+newdeb )

  if ret:
    S('xdelta delta -n -9 '+newdeb+' NEW/result.deb ')
  #S(['ar','qSc', 'temp.deb',]+ deltaparts)
  #s.write('xdelta patch '+o+'.xdelta'+' ../OLD/'+o+' '+o+'\n')
  
  s.close()
  S('gzip -9 PATCH/patch.sh')
d261 1
a261 1
  S(['ar','qSc', delta,]+ deltaparts)
d274 22
@
