vilimblog - Page 21 of 40 -

ndk-configure.sh

A way to run GNU autoconf configure scripts using the Android NDK platform headers and libraries, improving on this:

#!/bin/bash
#
# Go to the directory containing the configure script.
#
# Then just run something like:
#
EXAMPLE="~/android-ndk-r9 ./ndk-configure.sh android-9 arch-arm"

if [ -z "${NDK}" ]; then
    echo "Need to specify NDK environment variable when calling."
    echo "  -- e.g. ${EXAMPLE}"
    exit 1
fi

if [ "$#" -lt "2" ]; then
    echo "Usage: ./ndk-configure.sh <platform> <architecture>"
    echo "  -- e.g. ${EXAMPLE}"
    exit 1
else
    PLATFORM=$1
    shift

    ARCHITECTURE=$1
    shift

    if [ -x "config.guess" ]; then
	wget --output-file=config.guess http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD 
    fi

    if [ -x "config.sub" ]; then
	wget --output-file=config.sub   http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
    fi

    COMPILER_VERSION="4.8"

    export NDK_TOOLCHAIN=${NDK}/toolchains/arm-linux-androideabi-${COMPILER_VERSION}/prebuilt/linux-x86_64/bin
    export CROSS_COMPILE=arm-linux-androideabi

    export AR=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-ar
    export CC=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-gcc
    export CXX=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-g++
    export CXXCPP=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-cpp
    export LD=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-ld
    export NM=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-nm
    export OBJDUMP=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-objdump
    export RANLIB=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-ranlib
    export STRIP=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-strip

    export SYSROOT=${NDK}/platforms/${PLATFORM}/${ARCHITECTURE}                  # Needed for the Android-specific headers and libs.
    export CXX_SYSROOT=${NDK}/sources/cxx-stl/gnu-libstdc++/${COMPILER_VERSION}  # Needed for the STL headers and libs.
    export CXX_BITS_INCLUDE=${CXX_SYSROOT}/libs/armeabi/include                  # Needed for the <bits/c++config.h> and other headers.
                                                                                 # Certain STL classes, like <unordered_map> won't be 
                                                                                 # usable otherwise.

    export CPPFLAGS="-I${SYSROOT}/usr/include"
    export CFLAGS="--sysroot=${SYSROOT}"
    export CXXFLAGS="--std=c++11 --sysroot=${SYSROOT} -I${CXX_SYSROOT}/include -I${CXX_BITS_INCLUDE}"

    export LIBS="-lc"
    export LDFLAGS="-Wl,-rpath-link=${SYSROOT}/usr/lib -L${SYSROOT}/usr/lib -L${CXX_SYSROOT}/lib"

    export ac_cv_func_malloc_0_nonnull=yes
    export ac_cv_func_realloc_0_nonnull=yes
    
    MACHINE=$( ${CXX} -dumpmachine )

    echo "Machine:           ${MACHINE}"
    echo "Sysroot (Android): ${SYSROOT}"
    echo "Sysroot (cxx):     ${CXX_SYSROOT}"
    echo "Bits include:      ${CXX_BITS_INCLUDE}"

    argString=""
    
    while [ "$1" != "" ]; do
	argString="${argString} $1"
	shift
    done
    
    ./configure --host="${MACHINE}" --target="${MACHINE}" --with-sysroot="${SYSROOT}" $argString

fi

#!/bin/bash

# Go to the directory containing the configure script.

# Then just run something like:

EXAMPLE="~/android-ndk-r9 ./ndk-configure.sh android-9 arch-arm"

if [ -z "${NDK}" ]; then

echo "Need to specify NDK environment variable when calling."

echo " -- e.g. ${EXAMPLE}"

exit 1

if [ "$#" -lt "2" ]; then

echo "Usage: ./ndk-configure.sh <platform> <architecture>"

echo " -- e.g. ${EXAMPLE}"

exit 1

else

PLATFORM=$1

shift

ARCHITECTURE=$1

shift

if [ -x "config.guess" ]; then

wget --output-file=config.guess http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD

if [ -x "config.sub" ]; then

wget --output-file=config.sub http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD

COMPILER_VERSION="4.8"

export NDK_TOOLCHAIN=${NDK}/toolchains/arm-linux-androideabi-${COMPILER_VERSION}/prebuilt/linux-x86_64/bin

export CROSS_COMPILE=arm-linux-androideabi

export AR=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-ar

export CC=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-gcc

export CXX=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-g++

export CXXCPP=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-cpp

export LD=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-ld

export NM=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-nm

export OBJDUMP=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-objdump

export RANLIB=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-ranlib

export STRIP=${NDK_TOOLCHAIN}/${CROSS_COMPILE}-strip

export SYSROOT=${NDK}/platforms/${PLATFORM}/${ARCHITECTURE} # Needed for the Android-specific headers and libs.

export CXX_SYSROOT=${NDK}/sources/cxx-stl/gnu-libstdc++/${COMPILER_VERSION} # Needed for the STL headers and libs.

export CXX_BITS_INCLUDE=${CXX_SYSROOT}/libs/armeabi/include # Needed for the <bits/c++config.h> and other headers.

# Certain STL classes, like <unordered_map> won't be

# usable otherwise.

export CPPFLAGS="-I${SYSROOT}/usr/include"

export CFLAGS="--sysroot=${SYSROOT}"

export CXXFLAGS="--std=c++11 --sysroot=${SYSROOT} -I${CXX_SYSROOT}/include -I${CXX_BITS_INCLUDE}"

export LIBS="-lc"

export LDFLAGS="-Wl,-rpath-link=${SYSROOT}/usr/lib -L${SYSROOT}/usr/lib -L${CXX_SYSROOT}/lib"

export ac_cv_func_malloc_0_nonnull=yes

export ac_cv_func_realloc_0_nonnull=yes

MACHINE=$( ${CXX} -dumpmachine )

echo "Machine: ${MACHINE}"

echo "Sysroot (Android): ${SYSROOT}"

echo "Sysroot (cxx): ${CXX_SYSROOT}"

echo "Bits include: ${CXX_BITS_INCLUDE}"

argString=""

while [ "$1" != "" ]; do

argString="${argString} $1"

shift

done

./configure --host="${MACHINE}" --target="${MACHINE}" --with-sysroot="${SYSROOT}" $argString

References
1. http://stackoverflow.com/questions/5908581/is-hash-map-part-of-the-stl
2. http://protobuf.googlecode.com/svn/trunk/m4/stl_hash.m4
3. http://stackoverflow.com/questions/9201521/g-4-6-issue-no-bits-cconfig-h-file-as-required-by-the-header-cstring

Ubuntu 13.10 chroot on Android

Summary

Old smartphones make the perfect microservers, they’ve got more than enough horsepower for server tasks, plenty of RAM, and once rooted, full configurability. They’re durable embedded systems, honed to perfection by being produced for the mass-market, with no moving parts and the possibility of decent storage expansion. They’re extremely resource-light, usually utilizing only a few watt-hours per hour, meaning they’ve got battery-backup built right in.

Ubuntu 13.10 on an old smartphone

I had some problems getting various chroot instructions to work completely smoothly on my phone. This writeup is a synthesis of instructions from the references listed at the bottom of this post.

I used gparted to create a clean ext3-formatted Linux partition on an 8 gigabyte SD card. I mounted that partition on the host machine under /mnt and ran the debootstrap command:

HOSTNAME=ubuntu-armhf sudo debootstrap --arch=armhf --variant=minbase --foreign saucy /mnt

Then, I installed the rest of the system in emulation:

sudo apt-get install qemu-user-static qemu-system
sudo cp /usr/bin/qemu-arm-static /mnt/usr/bin
LANG=C sudo chroot mnt /usr/bin/qemu-arm-static -cpu cortex-a9 /bin/bash ./debootstrap/debootstrap --second-stage

After putting the SD card back into the smartphone, Android complained that the card was messed up:

Which was fixable by logging into the phone’s shell as root and running e2fsck, which checked and fixed filesystem issues it detected on the card:

localhost block # e2fsck /dev/block/mmcblk0p1
e2fsck 1.41.12 (17-May-2010)
chroot-drive contains a file system with errors, check forced.
Pass 1: Checking inodes, blocks, and sizes
Pass 2: Checking directory structure
Pass 3: Checking directory connectivity
Pass 4: Checking reference counts
Pass 5: Checking group summary information
chroot-drive: 8180/489600 files (1.2% non-contiguous), 100332/1957120 blocks
localhost block # e2fsck /dev/block/mmcblk0p1
e2fsck 1.41.12 (17-May-2010)
chroot-drive: clean, 8180/489600 files, 100332/1957120 blocks

After doing this once, the Android system checked the SD card again at the next reboot, and added the following mounts to the /proc/mounts table after each subsequent reboot:

/dev/block/vold/179:1 on /mnt/sdcard type ext4 (rw,dirsync,nosuid,nodev,noexec,relatime,barrier=1,data=ordered,noauto_da_alloc)
/dev/block/vold/179:1 on /mnt/secure/asec type ext4 (rw,dirsync,nosuid,nodev,noexec,relatime,barrier=1,data=ordered,noauto_da_alloc)
tmpfs on /mnt/sdcard/.android_secure type tmpfs (ro,relatime,size=0k,mode=000)

Then I set up a script to log into the chroot environment:

export SDCARD=/mnt/sdcard
export ROOT=$SDCARD
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
export HOME=/root
mount -o remount,exec,dev,suid $SDCARD
for f in dev dev/pts proc sys ; do mount -o bind /$f $ROOT/$f ; done
chroot $ROOT /bin/bash -l

export SDCARD=/mnt/sdcard

export ROOT=$SDCARD

export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH

export HOME=/root

mount -o remount,exec,dev,suid $SDCARD

for f in dev dev/pts proc sys ; do mount -o bind /$f $ROOT/$f ; done

chroot $ROOT /bin/bash -l

This worked fine, but when running inside the chroot environment the apt-get and apt-key commands didn’t work, because the Domain Name Servers were not specified. (Note: The mount command in the for loop probably also needs some work, as it will mount things multiple times when I leave and reenter the chroot environment.)

I edited the /etc/resolv.conf file and pointed it to Google’s nameservers:

# vi /mnt/sdcard/etc/resolv.conf

1	# vi /mnt/sdcard/etc/resolv.conf

nameserver 8.8.8.8
nameserver 8.8.4.4
search local

nameserver 8.8.8.8

nameserver 8.8.4.4

search local

The sources.list file uses ports.ubuntu.com, which might be because the other archive servers don’t have armhf packages to offer. It wasn’t quite clear to me how Ubuntu manages their ports across the various versions.

root@localhost:/# echo "deb http://ports.ubuntu.com saucy main restricted universe" > /etc/apt/sources.list
root@localhost:/# apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 428D7C01
Executing: gpg --ignore-time-conflict --no-options --no-default-keyring --secret-keyring /tmp/tmp.DuitQkXxTo --trustdb-name /etc/apt//trustdb.gpg --keyring /etc/apt/trusted.gpg --primary-keyring /etc/apt/trusted.gpg --keyserver keyserver.ubuntu.com --recv-keys 428D7C01
gpg: requesting key 428D7C01 from hkp server keyserver.ubuntu.com
gpg: key 428D7C01: public key "Ubuntu Debug Symbol Archive Automatic Signing Key <ubuntu-archive@lists.ubuntu.com>" imported
gpg: no ultimately trusted keys found
gpg: Total number processed: 1
gpg:               imported: 1

root@localhost:/# echo "deb http://ports.ubuntu.com saucy main restricted universe" > /etc/apt/sources.list

root@localhost:/# apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 428D7C01

Executing: gpg --ignore-time-conflict --no-options --no-default-keyring --secret-keyring /tmp/tmp.DuitQkXxTo --trustdb-name /etc/apt//trustdb.gpg --keyring /etc/apt/trusted.gpg --primary-keyring /etc/apt/trusted.gpg --keyserver keyserver.ubuntu.com --recv-keys 428D7C01

gpg: requesting key 428D7C01 from hkp server keyserver.ubuntu.com

gpg: key 428D7C01: public key "Ubuntu Debug Symbol Archive Automatic Signing Key <ubuntu-archive@lists.ubuntu.com>" imported

gpg: no ultimately trusted keys found

gpg: Total number processed: 1

gpg: imported: 1

Once the APT source has been added, apt-get update and apt-get install will work fine. It’s a blank slate from there.

squid-deb-proxy-client

For the initial downloads, the following setting can be placed in the APT configuration folder to manually specify an APT package proxy, which might speed up installations:

 
# vi /mnt/sdcard/etc/apt/apt.conf.d/30proxy
Acquire::http { Proxy "http://192.168.0.132:8000"; };

If the proxy address never changes, you can just leave it at that. If the address does change, you’ll probably want to install the squid-deb-proxy-client package.

References

1. ChrootOnAndroid
2. How to install Ubuntu 13.04 on your Android phone
3. DebootstrapChroot
4. Raspbian Benchmarking – armel vs armhf
5. Installation mit debootstrap
6. Index of /dists/saucy/main
7. Best way to cache apt downloads on a LAN?

C++11 support on Android with g++ 4.8

Summary

The C++11 support on Android when using g++ 4.8 is pretty good. The important concurrency features that were missing from g++ 4.6 are now available in the Standard Template Library.

Testing libgnustl_shared.so

After examining the LLVM libc++ library on Android, I wanted to check which C++11 features are currently available using g++ 4.8 and the GNU Standard Template Library (libstdc++). g++ 4.8 improved support for C++11 significantly (4.6 vs. 4.8), so the question was whether that support also made it into the Android-specific compilers. With the LLVM libc++ library, a lot of very specific platform code such as support for atomic operations was stubbed out, which would cascade down into problems with the mutual exclusion mechanisms and threads.

Based on a review of an existing codebase, the features most interesting to me were contained in the following headers: atomic, chrono, condition_variable (condition_variable), memory (shared_ptr, unique_ptr), mutex (lock_guard, mutex, unique_lock), thread (thread). Also of interest was compiler support for lambdas and auto keyword support.

So let’s see what’s available, by modifying the android-ndk-r9/samples/test-libstdc++ native executable to utilize a number of C++11 features to perform the same race condition-prone task. Namely, the program generates a random number of threads which each attempt to simultaneously increment some global variable by one, one hundred times.

C++ gives you a few ways to solve the problem, and I’ve tried each of them here. The logic of the incrementation isn’t exactly identical, and in some cases incredibly inefficient (the conditional_variable case basically has all threads waiting for a single thread to complete, before they can acquire a shared resource mutex and begin executing). In any case, this is more a proof of library completeness than a proof of algorithmic efficiency.

The test source code looks like this:

// New in C++ 11.

#include <atomic>
#include <chrono>
#include <condition_variable>
#include <memory>
#include <mutex>
#include <random>
#include <thread>

// Regular STL.

#include <list>
#include <string>

// C-compatibility.

#include <cassert>
#include <cerrno>
#include <cstdio>
#include <cstddef>


// Constants.

const int INCREMENT = 100;

// Utility functions (std::random)

std::random_device         randomDevice;
std::default_random_engine randomEngine(randomDevice());

int diceRoll(int sides = 20)
{
    std::uniform_int_distribution<int> distribution(1, sides);

    int roll = distribution(randomEngine);
    printf("  -- std::random rolled: %d\n", roll);

    return roll;
}

// std::atomic

std::atomic<int> atomicInt(0);

void stdAtomicThreadFn(void)
{
    // Try creating a bunch of threads, each one interested
    // only in incrementing the global atomicInt.

    // If it's truly atomic, there shouldn't be any race
    // conditions and the final int value should be an
    // exact multiple of this thread's loop.

    for (int i = 0; i < INCREMENT; ++i)
    {
        ++atomicInt;
    }
}

// std::mutex + std::condition_variable + std::unique_lock + lambda tests

int        nonAtomicInt = 0;
bool       nonAtomicThreadsReady = false;

std::mutex              nonAtomicIntMutex;
std::condition_variable nonAtomicIntCv;

void cvMutexThreadFn(void)
{
    // Use the mutex and signalling capabilities in the 
    // Standard Library to safely increment the nonAtomicInt.

    // Start the thread but wait on a condition variable
    // before starting the nonAtomicInt incrementation.

    std::unique_lock<std::mutex> lock(nonAtomicIntMutex);
    nonAtomicIntCv.wait(lock, [] { return nonAtomicThreadsReady; });

    // This runs after the nonAtomicIntMutex lock is reacquired,
    // therefore only a single thread at a time is modifying
    // nonAtomicInt.

    for (int i = 0; i < INCREMENT; ++i)
    {
        nonAtomicInt++;
    }
}

// std::lock_guard + std::mutex tests

int        lockGuardInt = 0;
std::mutex lockGuardIntMutex;

void incrementLockGuardInt(void)
{
    std::lock_guard<std::mutex> lock(lockGuardIntMutex);

    lockGuardInt++;
}

// std::unique_ptr, std::shared_ptr, and std::make_shared test class.

class PtrTest
{
private:
    std::string ptrType;
public:
    PtrTest(std::string const& ptrType) : ptrType(ptrType) { printf("  --  PtrTest(): %s\n", ptrType.c_str()); };
   ~PtrTest()                                              { printf("  -- ~PtrTest(): %s\n", ptrType.c_str()); };
};


void testA(void)
{
    printf("std::atomic + std::random + std::thread + auto + ranged-for test\n");
    
    std::list<std::thread> atomicIntThreads;
    const int atomicIntThreadCount = diceRoll();

    // Create a number of threads, all incrementing atomicInt.

    for (int i = 0; i < atomicIntThreadCount; ++i)
    {
        atomicIntThreads.push_back(std::thread(&stdAtomicThreadFn));
    }

    // Join those threads.

    for (auto & oneThread : atomicIntThreads)
    {
        oneThread.join();
    }

    assert(atomicInt == atomicIntThreadCount * INCREMENT);
    printf("  -- atomicInt final value: %d\n", atomicInt.load());
}


void testB(void)
{
    printf("std::condition_variable + std::mutex + std::unique_lock + std::random + std::thread + lambda test\n");

    std::list<std::thread> nonAtomicIntThreads;
    const int nonAtomicIntThreadCount = diceRoll();

    // Create a number of threads, all incrementing nonAtomicInt.

    for (int i = 0; i < nonAtomicIntThreadCount; ++i)
    {
        nonAtomicIntThreads.push_back(std::thread(&cvMutexThreadFn));
    }

    // Start those threads.
    nonAtomicThreadsReady = true;
    nonAtomicIntCv.notify_all();

    // Join those threads.

    for (auto & oneNonAtomicThread : nonAtomicIntThreads)
    {
        oneNonAtomicThread.join();
    }

    assert(nonAtomicInt == nonAtomicIntThreadCount * INCREMENT);
    printf("  -- nonAtomicInt final value: %d\n", nonAtomicInt);
}

void testC(void)
{
    printf("std::lock_guard + std::mutex + std::random + std::thread + lambda test\n");

    std::list<std::thread> lockGuardThreads;
    const int lockGuardThreadCount = diceRoll();

    // Create a number of threads, all incrementing lockGuardInt.

    for (int i = 0; i < lockGuardThreadCount; ++i)
    {
        lockGuardThreads.push_back(std::thread([] {
            for (int j = 0; j < INCREMENT; ++j) incrementLockGuardInt();
        }));
    }

    // Join those threads.

    for (auto & oneLockGuardThread : lockGuardThreads)
    {
        oneLockGuardThread.join();
    }

    assert(lockGuardInt == lockGuardThreadCount * INCREMENT);
    printf("  -- lockGuardInt final value: %d\n", lockGuardInt);
}

void testD(void)
{
    printf("std::unique_ptr + std::shared_ptr + std::make_shared tests\n");

    std::unique_ptr<PtrTest> uP(new PtrTest("std::unique_ptr"));
    std::shared_ptr<PtrTest> sP = std::make_shared<PtrTest>("std::shared_ptr");
}


int main(void)
{
    testA();
    testB();
    testC();
    testD();

    return 0;
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

// New in C++ 11.

#include <atomic>

#include <chrono>

#include <condition_variable>

#include <memory>

#include <mutex>

#include <random>

#include <thread>

// Regular STL.

#include <list>

#include <string>

// C-compatibility.

#include <cassert>

#include <cerrno>

#include <cstdio>

#include <cstddef>

// Constants.

const int INCREMENT = 100;

// Utility functions (std::random)

std::random_device randomDevice;

std::default_random_engine randomEngine(randomDevice());

int diceRoll(int sides = 20)

{

std::uniform_int_distribution<int> distribution(1, sides);

int roll = distribution(randomEngine);

printf(" -- std::random rolled: %d\n", roll);

return roll;

}

// std::atomic

std::atomic<int> atomicInt(0);

void stdAtomicThreadFn(void)

{

// Try creating a bunch of threads, each one interested

// only in incrementing the global atomicInt.

// If it's truly atomic, there shouldn't be any race

// conditions and the final int value should be an

// exact multiple of this thread's loop.

for (int i = 0; i < INCREMENT; ++i)

{

++atomicInt;

}

// std::mutex + std::condition_variable + std::unique_lock + lambda tests

int nonAtomicInt = 0;

bool nonAtomicThreadsReady = false;

std::mutex nonAtomicIntMutex;

std::condition_variable nonAtomicIntCv;

void cvMutexThreadFn(void)

{

// Use the mutex and signalling capabilities in the

// Standard Library to safely increment the nonAtomicInt.

// Start the thread but wait on a condition variable

// before starting the nonAtomicInt incrementation.

std::unique_lock<std::mutex> lock(nonAtomicIntMutex);

nonAtomicIntCv.wait(lock, [] { return nonAtomicThreadsReady; });

// This runs after the nonAtomicIntMutex lock is reacquired,

// therefore only a single thread at a time is modifying

// nonAtomicInt.

for (int i = 0; i < INCREMENT; ++i)

{

nonAtomicInt++;

}

// std::lock_guard + std::mutex tests

int lockGuardInt = 0;

std::mutex lockGuardIntMutex;

void incrementLockGuardInt(void)

{

std::lock_guard<std::mutex> lock(lockGuardIntMutex);

lockGuardInt++;

}

// std::unique_ptr, std::shared_ptr, and std::make_shared test class.

class PtrTest

{

private:

std::string ptrType;

public:

PtrTest(std::string const& ptrType) : ptrType(ptrType) { printf(" -- PtrTest(): %s\n", ptrType.c_str()); };

~PtrTest() { printf(" -- ~PtrTest(): %s\n", ptrType.c_str()); };

};

void testA(void)

{

printf("std::atomic + std::random + std::thread + auto + ranged-for test\n");

std::list<std::thread> atomicIntThreads;

const int atomicIntThreadCount = diceRoll();

// Create a number of threads, all incrementing atomicInt.

for (int i = 0; i < atomicIntThreadCount; ++i)

{

atomicIntThreads.push_back(std::thread(&stdAtomicThreadFn));

}

// Join those threads.

for (auto & oneThread : atomicIntThreads)

{

oneThread.join();

}

assert(atomicInt == atomicIntThreadCount * INCREMENT);

printf(" -- atomicInt final value: %d\n", atomicInt.load());

}

void testB(void)

{

printf("std::condition_variable + std::mutex + std::unique_lock + std::random + std::thread + lambda test\n");

std::list<std::thread> nonAtomicIntThreads;

const int nonAtomicIntThreadCount = diceRoll();

// Create a number of threads, all incrementing nonAtomicInt.

for (int i = 0; i < nonAtomicIntThreadCount; ++i)

{

nonAtomicIntThreads.push_back(std::thread(&cvMutexThreadFn));

}

// Start those threads.

nonAtomicThreadsReady = true;

nonAtomicIntCv.notify_all();

// Join those threads.

for (auto & oneNonAtomicThread : nonAtomicIntThreads)

{

oneNonAtomicThread.join();

}

assert(nonAtomicInt == nonAtomicIntThreadCount * INCREMENT);

printf(" -- nonAtomicInt final value: %d\n", nonAtomicInt);

}

void testC(void)

{

printf("std::lock_guard + std::mutex + std::random + std::thread + lambda test\n");

std::list<std::thread> lockGuardThreads;

const int lockGuardThreadCount = diceRoll();

// Create a number of threads, all incrementing lockGuardInt.

for (int i = 0; i < lockGuardThreadCount; ++i)

{

lockGuardThreads.push_back(std::thread([] {

for (int j = 0; j < INCREMENT; ++j) incrementLockGuardInt();

}));

}

// Join those threads.

for (auto & oneLockGuardThread : lockGuardThreads)

{

oneLockGuardThread.join();

}

assert(lockGuardInt == lockGuardThreadCount * INCREMENT);

printf(" -- lockGuardInt final value: %d\n", lockGuardInt);

}

void testD(void)

{

printf("std::unique_ptr + std::shared_ptr + std::make_shared tests\n");

std::unique_ptr<PtrTest> uP(new PtrTest("std::unique_ptr"));

std::shared_ptr<PtrTest> sP = std::make_shared<PtrTest>("std::shared_ptr");

}

int main(void)

{

testA();

testB();

testC();

testD();

return 0;

}

I added an Application.mk file to the jni folder. Debugging is activated, so that the assert() macro isn’t disabled.

APP_ABI    := armeabi
APP_STL    := gnustl_shared
APP_CFLAGS := --std=c++11
APP_OPTIM  := debug

NDK_TOOLCHAIN_VERSION := 4.8

Building works fine:

$ ndk-build
Compile++ thumb  : test-libstl /sources/cxx-stl/gnu-libstdc++/4.8/libs/armeabi/
Executable     : test-libstl
Install        : test-libstl => libs/armeabi/test-libstl
Install        : libgnustl_shared.so => libs/armeabi/libgnustl_shared.so
$ adb push libs/armeabi/libgnustl_shared.so /data/tmp
366 KB/s (804484 bytes in 2.143s)
$ adb push libs/armeabi/test-libstl /data/tmp
284 KB/s (50536 bytes in 0.173s)

After pushing the test-libstl executable to the device, make sure the /data/tmp folder is added to the LD_LIBRARY_PATH:

localhost tmp # LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/tmp

Then just run the executable, and you should see:

localhost tmp # ./test-libstl 
std::atomic + std::random + std::thread + auto + ranged-for test
  -- std::random rolled: 11
  -- atomicInt final value: 1100
std::condition_variable + std::mutex + std::unique_lock + std::random + std::thread + lambda test
  -- std::random rolled: 15
  -- nonAtomicInt final value: 1500
std::lock_guard + std::mutex + std::random + std::thread + lambda test
  -- std::random rolled: 7
  -- lockGuardInt final value: 700
std::unique_ptr + std::shared_ptr + std::make_shared tests
  --  PtrTest(): std::unique_ptr
  --  PtrTest(): std::shared_ptr
  -- ~PtrTest(): std::shared_ptr
  -- ~PtrTest(): std::unique_ptr

To run the tests, I used a rooted phone running Android 2.3.7.

In real deployments where you’re building a loadable shared-library using the Java Native Interface, you’ll need to follow the instructions from the NDK’s CPLUSPLUS-SUPPORT.html file and call System.loadLibrary on libgnustl_shared.so before loading the eventual shared library form of your app. (Managing standalone executable lifecycles tends to be a pain in the ass inside of Android apps and services, at least based on the experiences we had at my previous company.)

Overall, I’m pretty positive about the support for concurrency in g++ 4.8 on Android, and am thrilled to see that it may now be possible to write a single modern C++ codebase across all of the major desktop and mobile platforms.

Mucking around with libc++ on Android

tl;dr

libc++ on Android doesn’t work yet as of October 2013.

Getting the test suite running

The LLVM project-created libc++ library isn’t yet fully supported on Android, but I wanted to see how much of its test suite would currently work on an Android phone. Considering the weaknesses of all the other C++ Standard Template Libraries out there, libc++ becoming available would be a huge boon to developing a modern codebase. There are a few notes about this in one long-running feature request, and according to the git logs there are maybe 2 people actually working full-time on the NDK at Google, but there’s not much else to note.

Easier said than done, as usual, and it’s not really in Google’s interest to get this working, since everyone is supposed to write things specificially in Java. But it is the one stumbling block that prevents a company from writing their core logic a single time in modern C++11 (with conveniences like support for atomics, threads, conditional variables, and mutexes) and having that run across all existing desktop and mobile platforms. I’ll have to take a look at whether the g++ 4.8 STL support is complete enough to cover the C++11 cases I want to use. Everyone else (Microsoft, Apple) has a usable STL implementation on their desktop and mobile platforms. A functioning libc++ might also let people avoid pulling in the entirety of a library like Qt to plug the gaps, and as it is Qt is in an alpha stage of support for Android and iOS.

Anyway, these are some of the notes I took while getting the test suite going on a rooted smartphone.

Clone the latest Google NDK repository sources in $HOME:

git clone https://android.googlesource.com/platform/ndk git-ndk

Grab a copy of the prepackaged NDK from the Android developer website, at the time of this writing version r9. Unpack this in $HOME.

Create a symbolic link from sources/cxx-stl/llvm-libc++ folder in the Git checkout to the sources/cxx-stl folder in the officially packaged NDK folder. This saves you from having to build the cross-compilers yourself.

ln -s git-ndk/sources/cxx-stl/llvm-libc++ android-ndk-r9/sources/cxx-stl/llvm-libc++

Edit the android-ndk-r9/sources/cxx-stl/llvm-libc++/test/../../gabi++/include/gabixx_config.h file to use the __typeof__ operator instead of typeof, because of Problems 3 and 4 (detailed below) encountered in --std=c++11 mode:

// TODO(digit): Use __atomic_load_acq_rel when available.
#define __gabixx_sync_load(address)  \
    __sync_fetch_and_add((address), (__typeof__(*(address)))0)

// Clang provides __sync_swap(), but GCC does not.
// IMPORTANT: For GCC, __sync_lock_test_and_set has acquire semantics only
// so an explicit __sync_synchronize is needed to ensure a full barrier.
// TODO(digit): Use __atomic_swap_acq_rel when available.
#if defined(__clang__)
#  define __gabixx_sync_swap(address,value)  __sync_swap((address),(value))
#else
#  define __gabixx_sync_swap(address, value)  \
  __extension__ ({ \
    __typeof__(*(address)) __ret = __sync_lock_test_and_set((address),(value)); \
    __sync_synchronize(); \
    __ret; \
  })
#endif

// TODO(digit): Use __atomic_load_acq_rel when available.

#define __gabixx_sync_load(address) \

__sync_fetch_and_add((address), (__typeof__(*(address)))0)

// Clang provides __sync_swap(), but GCC does not.

// IMPORTANT: For GCC, __sync_lock_test_and_set has acquire semantics only

// so an explicit __sync_synchronize is needed to ensure a full barrier.

// TODO(digit): Use __atomic_swap_acq_rel when available.

#if defined(__clang__)

# define __gabixx_sync_swap(address,value) __sync_swap((address),(value))

#else

# define __gabixx_sync_swap(address, value) \

__extension__ ({ \

__typeof__(*(address)) __ret = __sync_lock_test_and_set((address),(value)); \

__sync_synchronize(); \

__ret; \

})

#endif

After making this change, go to the android-ndk-r9/tests/device/test-libc++ folder and try building the test libc++ executable, but limit the build to armeabi, otherwise the ndk-build will try building 8 versions of the libc++ libraries (static, shared) x (armeabi, armeabi-v7a, mips, x86):

android-ndk-r9/tests/device/test-libc++$ ndk-build APP_ABI=armeabi

The code compiles, but there’s still a ton of stuff in the library not yet implemented, and you’ll see a lot of #warning Not implemented messages, wherever functionality has been stubbed out.

[...]
Compile++ thumb  : c++_static  libs/armeabi/test_1_static

OK, so the library builds, now to get the testit_android test runner running in android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test:

Set up the libc++ library include path to look like:

LIBCXX_LIBS=$(cd $LIBCXX_ROOT/../../../.. && pwd)/tests/device/test-libc++/obj/local/$TARGET_ABI

1	LIBCXX_LIBS=$(cd $LIBCXX_ROOT/../../../.. && pwd)/tests/device/test-libc++/obj/local/$TARGET_ABI

The sysroot setting needs to be fixed up in the testit_android script, in this case I set it to the android-9 platform, which reaches back to roughly the Android 2.2 release:

ANDROID_SUPPORT=$(cd "$LIBCXX_ROOT"/../../../android/support && pwd)
ANDROID_SYSROOT=$(cd "$ANDROID_SUPPORT"/../../../platforms/android-9/arch-arm && pwd)

1 2	ANDROID_SUPPORT=$(cd "$LIBCXX_ROOT"/../../../android/support && pwd) ANDROID_SYSROOT=$(cd "$ANDROID_SUPPORT"/../../../platforms/android-9/arch-arm && pwd)

Add ANDROID_SYSROOT to the two compilation commands:

if run $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o $TEST_EXE $ANDROID_SYSROOT > /dev/null 2>&1

1	if run $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o $TEST_EXE $ANDROID_SYSROOT > /dev/null 2>&1

Now connect a rooted Android phone to your development computer.

Running testit_android should work, first pushing a copy of the libc++ shared library to the smartphone, then compiling, pushing, and running the test executables on the smartphone:

android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test$ ./testit_android --abi=armeabi
passed 1 tests in /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test
passed 1 tests in /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/algorithms
passed 1 tests in /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/algorithms/alg.c.library
passed 1 tests in /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/algorithms/alg.modifying.operations
[...]

Unfortunately, after running several thousand tests for over an hour, testit_android seemed to hang when attempting to run tests within the support.dynamic test suite:

[...]
failed 1 tests in /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/language.support/support.dynamic/alloc.errors/set.new.handler
passed 1 tests in /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/language.support/support.dynamic/alloc.errors/set.new.handler

real	68m19.512s
user	0m0.004s
sys	0m0.000s

I removed the llvm-libc++/libcxx/test/support.dynamic test suite and reran testit_android, which resulted in the full suite completing:

****************************************************
Results for /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test:
using arm-linux-androideabi-g++ (GCC) 4.8
Copyright (C) 2013 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
with -std=c++11 -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/support -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/include -I/home/builder/android-ndk-r9/sources/android/support/include -L/home/builder/android-ndk-r9/tests/device/test-libc++/obj/local/armeabi
----------------------------------------------------
sections without tests   : 0
sections with failures   : 199
sections without failures: 861
                       +   ----
total number of sections : 1060
----------------------------------------------------
number of tests failed   : 683
number of tests passed   : 3716
                       +   ----
total number of tests    : 4399
****************************************************

real	141m23.980s
user	67m18.716s
sys	21m46.932s

As usual, your mileage may vary, but this is the state of libc++ on Android in October 2013.

Notes

1. How about using Clang, since it should work better?

According to the NDK page, you can specify the compiler to use via the NDK_TOOLCHAIN_VERSION variable:

“Fixed the ndk-build script to enable you to specify a version of Clang as a command line option (e.g., NDK_TOOLCHAIN_VERSION=clang3.2). Previously, only specifying the version as an environment variable worked.”

So this works too:

ndk-build NDK_TOOLCHAIN_VERSION=clang3.3 APP_ABI=armeabi

And produces output like:

[...]
Compile++ thumb  : c++_shared 
The problem is that the testit_android script is currently hard-coded to use arm-linux-androideabi-g++ to build the test suite executables. The LLVM compiler that ships with Android NDK r9 seems to be an all-in-one cross-compiler, which can generate code for ARM, MIPS, and x86 depending on which config.mk files are supplied to configure the current build. So getting testit_android to build ARM executables with Clang would be a bit more work, though probably not that much more work.
2. Relation between the Android.mk files currently
android-ndk-r9/tests/device/test-libc++/jni/Android.mk (immediately imports test executable makefile) -> 
android-ndk-r9/sources/cxx-stl/llvm-libc++/test/Android.mk (builds the test executables) -> 
android-ndk-r9/sources/cxx-stl/llvm-libc++/Android.mk (builds the actual libc++ static and shared libraries)

Problems encountered
1. Attempting to build the tests in the Git working directory (git-ndk) without first creating the symbolic link in the release NDK directory (android-ndk-r9)
git-ndk/tests/device/test-libc++$ ndk-build
jni/Android.mk:1: /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/test/Android.mk: No such file or directory
jni/Android.mk:1: /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/test/Android.mk: No such file or directory
jni/Android.mk:1: /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/test/Android.mk: No such file or directory
jni/Android.mk:1: /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/test/Android.mk: No such file or directory
/home/builder/android-ndk-r9/build/core/build-all.mk:89: Android NDK: WARNING: There are no modules to build in this project!    
make: *** No rule to make target `/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/test/Android.mk'.  Stop.

2. Attempting to build the tests by specifying NDK_ROOT with ndk-build on the PATH
git-ndk/tests/device/test-libc++$ ndk-build NDK_ROOT=$HOME/git-ndk
Android NDK: Could not find platform files (headers and libraries)    
Android NDK: Please run build/tools/build-platforms.sh to build the corresponding directory.    
/home/builder/git-ndk/build/core/init.mk:471: *** Android NDK: Aborting    .  Stop.

Problems 1 and 2 are solved by creating the symbolic link as described above and running the ndk-build command from within the android-ndk-r9 folder structure. 
3. Attempting to build with GNU g++ fails due to the use in GAbi++ of a GNU-specific 'typeof' keyword which isn't recognized in --std=c++11 mode
Compile++ thumb  : c++_shared 
4. Attempting to build with Clang 3.3 fails due to the use in GAbi++ of a GNU-specific 'typeof' keyword which isn't recognized in --std=c++11 mode
Compile++ thumb  : c++_shared 
typeof is a GNU compiler-specific keyword extension to the C standard, meaning that other compilers won't like it very much. Darn it. The compilation commands for g++ and clang++ use --std=c++11, which does not allow the use of the typeof keyword. 
However, according to the Clang user manual:
The parser recognizes "asm" and "typeof" as keywords in gnu* modes; the variants "__asm__" and "__typeof__" are recognized in all modes.
The same appears to be true of the GNU compiler, therefore replacing typeof with __typeof__ fixes the problem on both.
Running testit_android
1. Missing prebuilt library
android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test$ ./testit_android --abi=armeabi
ERROR: Missing prebuilt library: /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libs/armeabi/libc++_static.a
Please run: build/tools/build-cxx-stl.sh --stl=libc++

But the library is already built here:
android-ndk-r9/sources/cxx-stl/llvm-libc++/libs/armeabi/libc++_static.a

So, the testit_android script just needs to be modified, where the line:
LIBCXX_LIBS=$(cd $LIBCXX_ROOT/.. && pwd)/libs/$TARGET_ABI

Needs to be changed to:
LIBCXX_LIBS=$(cd $LIBCXX_ROOT/../../../.. && pwd)/tests/device/test-libc++/obj/local/$TARGET_ABI

2. Missing C++ compiler
./testit_android --abi=armeabi
ERROR: Missing C++ compiler: arm-linux-androideabi-g++

Now, put that compiler into the PATH:
export PATH=$HOME/android-ndk-r9/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64/bin:$PATH

3. Missing platform headers and libraries
The build command doesn't know which Android platform to compile and link against:
/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/containers/container.adaptors/queue/queue.defn/assign_move.pass.cpp failed to compile
Compile line was: ( cd /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/containers/container.adaptors/queue/queue.defn && arm-linux-androideabi-g++ -std=c++11 -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/support -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/include -I/home/builder/android-ndk-r9/sources/android/support/include -L/home/builder/android-ndk-r9/tests/device/test-libc++/obj/local/armeabi assign_move.pass.cpp -lc++_shared )

android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test$ cd /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/containers/container.adaptors/queue/queue.defn && arm-linux-androideabi-g++ -std=c++11 -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/support -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/include -I/home/builder/android-ndk-r9/sources/android/support/include -L/home/builder/android-ndk-r9/tests/device/test-libc++/obj/local/armeabi assign_move.pass.cpp -lc++_shared
In file included from /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/include/queue:168:0,
                 from assign_move.pass.cpp:14:
/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/include/__config:99:21: fatal error: endian.h: No such file or directory
 # include <endian.h>
                     ^
compilation terminated.

android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test$ cd /home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/algorithms/alg.modifying.operations/alg.copy && arm-linux-androideabi-g++ -std=c++11 -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/test/support -I/home/builder/android-ndk-r9/sources/cxx-stl/llvm-libc++/libcxx/include -I/home/builder/android-ndk-r9/sources/android/support/include -I/home/builder/android-ndk-r9/sources/android/support/../../../platforms/android-9/arch-arm/usr/include -L/home/builder/android-ndk-r9/tests/device/test-libc++/obj/local/armeabi copy_n.pass.cpp -lc++_shared
/home/builder/android-ndk-r9/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64/bin/../lib/gcc/arm-linux-androideabi/4.8/../../../../arm-linux-androideabi/bin/ld: error: cannot open crtbegin_dynamic.o: No such file or directory
/home/builder/android-ndk-r9/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64/bin/../lib/gcc/arm-linux-androideabi/4.8/../../../../arm-linux-androideabi/bin/ld: error: cannot open crtend_android.o: No such file or directory
/home/builder/android-ndk-r9/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64/bin/../lib/gcc/arm-linux-androideabi/4.8/../../../../arm-linux-androideabi/bin/ld: error: cannot find -lstdc++
/home/builder/android-ndk-r9/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64/bin/../lib/gcc/arm-linux-androideabi/4.8/../../../../arm-linux-androideabi/bin/ld: error: cannot find -lm
/home/builder/android-ndk-r9/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64/bin/../lib/gcc/arm-linux-androideabi/4.8/../../../../arm-linux-androideabi/bin/ld: error: cannot find -lc
/home/builder/android-ndk-r9/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64/bin/../lib/gcc/arm-linux-androideabi/4.8/../../../../arm-linux-androideabi/bin/ld: error: cannot find -ldl
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, output_iterator >(): error: undefined reference to 'memset'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, output_iterator >(): error: undefined reference to '__assert2'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, output_iterator >(): error: undefined reference to '__assert2'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, input_iterator >(): error: undefined reference to 'memset'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, input_iterator >(): error: undefined reference to '__assert2'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, input_iterator >(): error: undefined reference to '__assert2'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, forward_iterator >(): error: undefined reference to 'memset'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test, bidirectional_iterator >(): error: undefined reference to 'memset'
/tmp/cc55fvWI.o:copy_n.pass.cpp:function void test(): error: undefined reference to 'memmove'
collect2: error: ld returned 1 exit status

As specified above, this problem is fixed by the addition of the ANDROID_SYSROOT variable to the build command, to indicate which Android platform includes and libraries the compiler should use.

Adding Python Twisted module to a virtualenv on Windows

I’m trying to keep my Python system-level install as clean as possible on Windows, but also trying to get Buildbot set up in a virtualenv. Unfortunately, some of the Buildbot dependencies want to build native extensions, and the most important — Twisted — just fails.

This is all I have in my system-level site-packages:

c:\>pip freeze
distribute==0.6.35
ipython==0.13.1
pyreadline==1.7.1
virtualenv==1.8.4

c:\>pip freeze

distribute==0.6.35

ipython==0.13.1

pyreadline==1.7.1

virtualenv==1.8.4

If I create a new virtualenv with --no-site-packages, then try to install the buildbot-slave package, I get the dreaded "Unable to find vcvarsall.bat" when Twisted is being installed:

copying twisted\test\raiser.c -> build\lib.win32-2.7\twisted\test
running build_ext
error: Unable to find vcvarsall.bat

copying twisted\test\raiser.c -> build\lib.win32-2.7\twisted\test

running build_ext

error: Unable to find vcvarsall.bat

Unfortunately, if you download and attempt to install the Twisted redistributable package from their website directly into a virtualenv, the installer package will register itself with the system, even if you select the “Install just for me” option. In other words, you can’t just have the package install itself portably into multiple virtualenvs:

Select “Install just for me”
Set “Python 2.7 from registry” to “Entire feature will be unavailable”
Set “Python from another location” to “Will be installed on local hard drive”
Set “Provide an alternate Python location” to the location of your virtualenv "c:\virtualenv"

The User Account Control (UAC) dialog will pop up, and after the install, you’ll be stuck with a single install of Twisted which can only be Repaired or Removed:

This situation kind of sucks. Unfortunately, the .exe version of the installer also won’t let you choose where to install its contents. It just gets stuck on the dialog where it autodetects your existing Python 2.7 installation, but won’t let you choose a different site-packages folder as a file destination. It only goes for the global install.

The solution is to download the .exe version of the installer, unpack it using something like 7-zip, and then copy the files from the PLATLIB folder to your virtualenv’s Libs\site-packages and the the files from the SCRIPTS folder to your virtualenv’s Scripts folder.

Then, the next time you run pip freeze, you should see something like:

(virtualenv) C:\virtualenv>pip freeze
Twisted==13.1.0

1 2	(virtualenv) C:\virtualenv>pip freeze Twisted==13.1.0

Then you can install the rest of the Buildbot packages:

(virtualenv) c:\virtualenv>pip install sqlalchemy==0.7.10
Downloading/unpacking sqlalchemy==0.7.10
  Downloading SQLAlchemy-0.7.10.tar.gz (3.5MB): 3.5MB downloaded
  Running setup.py egg_info for package sqlalchemy

    warning: no files found matching '*.jpg' under directory 'doc'
    no previously-included directories found matching 'doc\build\output'
Installing collected packages: sqlalchemy
  Running setup.py install for sqlalchemy
    building 'sqlalchemy.cprocessors' extension
    ***************************************************************************
    Unable to find vcvarsall.bat
    WARNING: The C extension could not be compiled, speedups are not enabled.
    Failure information, if any, is above.
    Retrying the build without the C extension now.
    ***************************************************************************

    warning: no files found matching '*.jpg' under directory 'doc'
    no previously-included directories found matching 'doc\build\output'
    ***************************************************************************
    WARNING: The C extension could not be compiled, speedups are not enabled.
    Plain-Python build succeeded.
    ***************************************************************************
Successfully installed sqlalchemy
Cleaning up...

(virtualenv) c:\virtualenv>pip install buildbot-slave
Downloading/unpacking buildbot-slave
  Downloading buildbot-slave-0.8.8.zip (164kB): 164kB downloaded
  Running setup.py egg_info for package buildbot-slave

Requirement already satisfied (use --upgrade to upgrade): twisted>=8.0.0 in c:\virtualenv\lib\site-packages (from buildbot-slave)
Downloading/unpacking zope.interface>=3.6.0 (from twisted>=8.0.0->buildbot-slave)
  Downloading zope.interface-4.0.5.zip (173kB): 173kB downloaded
  Running setup.py egg_info for package zope.interface

    warning: no previously-included files matching '*.dll' found anywhere in distribution
    warning: no previously-included files matching '*.pyc' found anywhere in distribution
    warning: no previously-included files matching '*.pyo' found anywhere in distribution
    warning: no previously-included files matching '*.so' found anywhere in distribution
Requirement already satisfied (use --upgrade to upgrade): setuptools in c:\virtualenv\lib\site-packages\setuptools-0.6c11-py2.7.egg (from
 zope.interface>=3.6.0->twisted>=8.0.0->buildbot-slave)
Installing collected packages: buildbot-slave, zope.interface
  Running setup.py install for buildbot-slave

  Running setup.py install for zope.interface

    warning: no previously-included files matching '*.dll' found anywhere in distribution
    warning: no previously-included files matching '*.pyc' found anywhere in distribution
    warning: no previously-included files matching '*.pyo' found anywhere in distribution
    warning: no previously-included files matching '*.so' found anywhere in distribution
    building 'zope.interface._zope_interface_coptimizations' extension
    ********************************************************************************
    WARNING:

            An optional code optimization (C extension) could not be compiled.

            Optimizations for this package will not be available!
    ()
    Unable to find vcvarsall.bat
    ********************************************************************************
    Skipping installation of c:\virtualenv\Lib\site-packages\zope\__init__.py (namespace package)
    Installing c:\virtualenv\Lib\site-packages\zope.interface-4.0.5-py2.7-nspkg.pth
Successfully installed buildbot-slave zope.interface
Cleaning up...

(virtualenv) c:\virtualenv>pip freeze
SQLAlchemy==0.7.10
Twisted==13.1.0
buildbot-slave==0.8.8
zope.interface==4.0.5

(virtualenv) c:\virtualenv>pip install sqlalchemy==0.7.10

Downloading/unpacking sqlalchemy==0.7.10

Downloading SQLAlchemy-0.7.10.tar.gz (3.5MB): 3.5MB downloaded

Running setup.py egg_info for package sqlalchemy

warning: no files found matching '*.jpg' under directory 'doc'

no previously-included directories found matching 'doc\build\output'

Installing collected packages: sqlalchemy

Running setup.py install for sqlalchemy

building 'sqlalchemy.cprocessors' extension

***************************************************************************

Unable to find vcvarsall.bat

WARNING: The C extension could not be compiled, speedups are not enabled.

Failure information, if any, is above.

Retrying the build without the C extension now.

***************************************************************************

warning: no files found matching '*.jpg' under directory 'doc'

no previously-included directories found matching 'doc\build\output'

***************************************************************************

WARNING: The C extension could not be compiled, speedups are not enabled.

Plain-Python build succeeded.

***************************************************************************

Successfully installed sqlalchemy

Cleaning up...

(virtualenv) c:\virtualenv>pip install buildbot-slave

Downloading/unpacking buildbot-slave

Downloading buildbot-slave-0.8.8.zip (164kB): 164kB downloaded

Running setup.py egg_info for package buildbot-slave

Requirement already satisfied (use --upgrade to upgrade): twisted>=8.0.0 in c:\virtualenv\lib\site-packages (from buildbot-slave)

Downloading/unpacking zope.interface>=3.6.0 (from twisted>=8.0.0->buildbot-slave)

Downloading zope.interface-4.0.5.zip (173kB): 173kB downloaded

Running setup.py egg_info for package zope.interface

warning: no previously-included files matching '*.dll' found anywhere in distribution

warning: no previously-included files matching '*.pyc' found anywhere in distribution

warning: no previously-included files matching '*.pyo' found anywhere in distribution

warning: no previously-included files matching '*.so' found anywhere in distribution

Requirement already satisfied (use --upgrade to upgrade): setuptools in c:\virtualenv\lib\site-packages\setuptools-0.6c11-py2.7.egg (from

zope.interface>=3.6.0->twisted>=8.0.0->buildbot-slave)

Installing collected packages: buildbot-slave, zope.interface

Running setup.py install for buildbot-slave

Running setup.py install for zope.interface

warning: no previously-included files matching '*.dll' found anywhere in distribution

warning: no previously-included files matching '*.pyc' found anywhere in distribution

warning: no previously-included files matching '*.pyo' found anywhere in distribution

warning: no previously-included files matching '*.so' found anywhere in distribution

building 'zope.interface._zope_interface_coptimizations' extension

********************************************************************************

WARNING:

An optional code optimization (C extension) could not be compiled.

Optimizations for this package will not be available!

()

Unable to find vcvarsall.bat

********************************************************************************

Skipping installation of c:\virtualenv\Lib\site-packages\zope\__init__.py (namespace package)

Installing c:\virtualenv\Lib\site-packages\zope.interface-4.0.5-py2.7-nspkg.pth

Successfully installed buildbot-slave zope.interface

Cleaning up...

(virtualenv) c:\virtualenv>pip freeze

SQLAlchemy==0.7.10

Twisted==13.1.0

buildbot-slave==0.8.8

zope.interface==4.0.5