#ifndef BOOST_DETAIL_ATOMIC_GCC_PPC_HPP
#define BOOST_DETAIL_ATOMIC_GCC_PPC_HPP

//  Copyright (c) 2009 Helge Bahmann
//
//  Distributed under the Boost Software License, Version 1.0.
//  See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)

#include <boost/atomic/detail/base.hpp>

/*
	Refer to: Motorola: "Programming Environments Manual for 32-Bit
	Implementations of the PowerPC Architecture", Appendix E:
	"Synchronization Programming Examples" for an explanation of what is
	going on here (can be found on the web at various places by the
	name "MPCFPE32B.pdf", Google is your friend...)
	
	Most of the atomic operations map to instructions in a relatively
	straight-forward fashion, but "load"s may at first glance appear
	a bit strange as they map to:
		
			lwz %rX, addr
			cmpw %rX, %rX
			bne- 1f
		1:
	
	That is, the CPU is forced to perform a branch that "formally" depends
	on the value retrieved from memory. This scheme has an overhead of
	about 1-2 clock cycles per load, but it allows to map "acquire" to
	the "isync" instruction instead of "sync" uniformly and for all type
	of atomic operations. Since "isync" has a cost of about 15 clock
	cycles, while "sync" hast a cost of about 50 clock cycles, the small
	penalty to atomic loads more than compensates for this.
	
	Byte- and halfword-sized atomic values are realized by encoding the
	value to be represented into a word, performing sign/zero extension
	as appropriate. This means that after add/sub operations the value
	needs fixing up to accurately preserve the wrap-around semantic of
	the smaller type. (Nothing special needs to be done for the bit-wise
	and the "exchange type" operators as the compiler already sees to
	it that values carried in registers are extended appropriately and
	everything falls into place naturally).
	
	The register constrant "b"  instructs gcc to use any register
	except r0; this is sometimes required because the encoding for
	r0 is used to signify "constant zero" in a number of instructions,
	making r0 unusable in this place. For simplicity this constraint
	is used everywhere since I am to lazy to look this up on a
	per-instruction basis, and ppc has enough registers for this not
	to pose a problem.
*/

#define BOOST_ATOMIC_CHAR_LOCK_FREE 2
#define BOOST_ATOMIC_CHAR16_T_LOCK_FREE 2
#define BOOST_ATOMIC_CHAR32_T_LOCK_FREE 2
#define BOOST_ATOMIC_WCHAR_T_LOCK_FREE 2
#define BOOST_ATOMIC_SHORT_LOCK_FREE 2
#define BOOST_ATOMIC_INT_LOCK_FREE 2
#define BOOST_ATOMIC_LONG_LOCK_FREE 2
#define BOOST_ATOMIC_ADDRESS_LOCK_FREE 2
#if defined(__powerpc64__)
#define BOOST_ATOMIC_LLONG_LOCK_FREE 2
#else
#define BOOST_ATOMIC_LLONG_LOCK_FREE 0
#endif
#define BOOST_ATOMIC_BOOL_LOCK_FREE 2

/* Would like to move the slow-path of failed compare_exchange
(that clears the "success" bit) out-of-line. gcc can in
principle do that using ".subsection"/".previous", but Apple's
binutils seemingly does not understand that. Therefore wrap
the "clear" of the flag in a macro and let it remain
in-line for Apple
*/

#if !defined(__APPLE__)

#define BOOST_ATOMIC_ASM_SLOWPATH_CLEAR \
	"1:\n" \
	".subsection 2\n" \
	"2: addi %1,0,0\n" \
	"b 1b\n" \
	".previous\n" \

#else

#define BOOST_ATOMIC_ASM_SLOWPATH_CLEAR \
	"b 1f\n" \
	"2: addi %1,0,0\n" \
	"1:\n" \

#endif

namespace boost {
namespace detail {
namespace atomic {

static inline void
ppc_fence_before(memory_order order)
{
	switch(order) {
		case memory_order_release:
		case memory_order_acq_rel:
#if defined(__powerpc64__)
			__asm__ __volatile__ ("lwsync" ::: "memory");
			break;
#endif
		case memory_order_seq_cst:
			__asm__ __volatile__ ("sync" ::: "memory");
		default:;
	}
}

static inline void
ppc_fence_after(memory_order order)
{
	switch(order) {
		case memory_order_acquire:
		case memory_order_acq_rel:
		case memory_order_seq_cst:
			__asm__ __volatile__ ("isync");
		case memory_order_consume:
			__asm__ __volatile__ ("" ::: "memory");
		default:;
	}
}

static inline void
ppc_fence_after_store(memory_order order)
{
	switch(order) {
		case memory_order_seq_cst:
			__asm__ __volatile__ ("sync");
		default:;
	}
}

/* integral types */

template<typename T>
class base_atomic<T, int, 1, true> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef int32_t storage_type;
	typedef T difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m"(v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ __volatile__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=&r" (v)
			: "m" (v_)
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"add %1,%0,%3\n"
			"extsb %1, %1\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"extsb %1, %1\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"and %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"or %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"xor %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};

template<typename T>
class base_atomic<T, int, 1, false> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef uint32_t storage_type;
	typedef T difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m"(v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ __volatile__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=&r" (v)
			: "m" (v_)
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"add %1,%0,%3\n"
			"rlwinm %1, %1, 0, 0xff\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"rlwinm %1, %1, 0, 0xff\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"and %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"or %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"xor %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};

template<typename T>
class base_atomic<T, int, 2, true> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef int32_t storage_type;
	typedef T difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m"(v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ __volatile__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=&r" (v)
			: "m" (v_)
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"add %1,%0,%3\n"
			"extsh %1, %1\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"extsh %1, %1\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"and %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"or %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"xor %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};

template<typename T>
class base_atomic<T, int, 2, false> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef uint32_t storage_type;
	typedef T difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m"(v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ __volatile__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=&r" (v)
			: "m" (v_)
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"add %1,%0,%3\n"
			"rlwinm %1, %1, 0, 0xffff\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"rlwinm %1, %1, 0, 0xffff\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"and %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"or %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"xor %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};

template<typename T, bool Sign>
class base_atomic<T, int, 4, Sign> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef T difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		const_cast<volatile value_type &>(v_) = v;
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v = const_cast<const volatile value_type &>(v_);
		__asm__ __volatile__ (
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "+b"(v)
			:
			: "cr0"
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"add %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"and %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"or %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"xor %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	value_type v_;
};

#if defined(__powerpc64__)

template<typename T, bool Sign>
class base_atomic<T, int, 8, Sign> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef T difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		const_cast<volatile value_type &>(v_) = v;
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v = const_cast<const volatile value_type &>(v_);
		__asm__ __volatile__ (
			"cmpd %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "+b"(v)
			:
			: "cr0"
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y1\n"
			"stdcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y2\n"
			"add %1,%0,%3\n"
			"stdcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"stdcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y2\n"
			"and %1,%0,%3\n"
			"stdcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y2\n"
			"or %1,%0,%3\n"
			"stdcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y2\n"
			"xor %1,%0,%3\n"
			"stdcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	value_type v_;
};

#endif

/* pointer types */

#if !defined(__powerpc64__)

template<bool Sign>
class base_atomic<void *, void *, 4, Sign> {
	typedef base_atomic this_type;
	typedef void * value_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m" (v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(v)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_BASE_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	value_type v_;
};

template<typename T, bool Sign>
class base_atomic<T *, void *, 4, Sign> {
	typedef base_atomic this_type;
	typedef T * value_type;
	typedef ptrdiff_t difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m" (v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(v)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile
	{
		v = v * sizeof(*v_);
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"add %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile
	{
		v = v * sizeof(*v_);
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"stwcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	value_type v_;
};

#else

template<bool Sign>
class base_atomic<void *, void *, 8, Sign> {
	typedef base_atomic this_type;
	typedef void * value_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"std %1, %0\n"
			: "+m" (v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ (
			"ld %0, %1\n"
			"cmpd %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(v)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y1\n"
			"stdcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_BASE_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	value_type v_;
};

template<typename T, bool Sign>
class base_atomic<T *, void *, 8, Sign> {
	typedef base_atomic this_type;
	typedef T * value_type;
	typedef ptrdiff_t difference_type;
public:
	explicit base_atomic(value_type v) : v_(v) {}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		ppc_fence_before(order);
		__asm__ (
			"std %1, %0\n"
			: "+m" (v_)
			: "r" (v)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		value_type v;
		__asm__ (
			"ld %0, %1\n"
			"cmpd %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(v)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		value_type original;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y1\n"
			"stdcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (v)
			: "cr0"
		);
		ppc_fence_after(order);
		return original;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected), "=&b" (success), "+Z"(v_)
			: "b" (expected), "b" (desired)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		return success;
	}
	
	value_type
	fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile
	{
		v = v * sizeof(*v_);
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y2\n"
			"add %1,%0,%3\n"
			"stdcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	value_type
	fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile
	{
		v = v * sizeof(*v_);
		value_type original, tmp;
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y2\n"
			"sub %1,%0,%3\n"
			"stdcx. %1,%y2\n"
			"bne- 1b\n"
			: "=&b" (original), "=&b" (tmp), "+Z"(v_)
			: "b" (v)
			: "cc");
		ppc_fence_after(order);
		return original;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	value_type v_;
};

#endif

/* generic */

template<typename T, bool Sign>
class base_atomic<T, void, 1, Sign> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef uint32_t storage_type;
public:
	explicit base_atomic(value_type v) : v_(0)
	{
		memcpy(&v_, &v, sizeof(value_type));
	}
	base_atomic(void) : v_(0) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp = 0;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m" (v_)
			: "r" (tmp)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		storage_type tmp;
		__asm__ __volatile__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(tmp)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		
		value_type v;
		memcpy(&v, &tmp, sizeof(value_type));
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp = 0, original;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (tmp)
			: "cr0"
		);
		ppc_fence_after(order);
		memcpy(&v, &original, sizeof(value_type));
		return v;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s = 0, desired_s = 0;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s = 0, desired_s = 0;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_BASE_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};

template<typename T, bool Sign>
class base_atomic<T, void, 2, Sign> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef uint32_t storage_type;
public:
	explicit base_atomic(value_type v) : v_(0)
	{
		memcpy(&v_, &v, sizeof(value_type));
	}
	base_atomic(void) : v_(0) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp = 0;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m" (v_)
			: "r" (tmp)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		storage_type tmp;
		__asm__ __volatile__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(tmp)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		
		value_type v;
		memcpy(&v, &tmp, sizeof(value_type));
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp = 0, original;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (tmp)
			: "cr0"
		);
		ppc_fence_after(order);
		memcpy(&v, &original, sizeof(value_type));
		return v;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s = 0, desired_s = 0;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s = 0, desired_s = 0;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_BASE_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};

template<typename T, bool Sign>
class base_atomic<T, void, 4, Sign> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef uint32_t storage_type;
public:
	explicit base_atomic(value_type v) : v_(0)
	{
		memcpy(&v_, &v, sizeof(value_type));
	}
	base_atomic(void) : v_(0) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp = 0;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"stw %1, %0\n"
			: "+m" (v_)
			: "r" (tmp)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		storage_type tmp;
		__asm__ __volatile__ (
			"lwz %0, %1\n"
			"cmpw %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(tmp)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		
		value_type v;
		memcpy(&v, &tmp, sizeof(value_type));
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp = 0, original;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"lwarx %0,%y1\n"
			"stwcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (tmp)
			: "cr0"
		);
		ppc_fence_after(order);
		memcpy(&v, &original, sizeof(value_type));
		return v;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s = 0, desired_s = 0;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s = 0, desired_s = 0;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: lwarx %0,%y2\n"
			"cmpw %0, %3\n"
			"bne- 2f\n"
			"stwcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_BASE_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};

#if defined(__powerpc64__)

template<typename T, bool Sign>
class base_atomic<T, void, 8, Sign> {
	typedef base_atomic this_type;
	typedef T value_type;
	typedef uint64_t storage_type;
public:
	explicit base_atomic(value_type v)
	{
		memcpy(&v_, &v, sizeof(value_type));
	}
	base_atomic(void) {}
	
	void
	store(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"std %1, %0\n"
			: "+m" (v_)
			: "r" (tmp)
		);
		ppc_fence_after_store(order);
	}
	
	value_type
	load(memory_order order = memory_order_seq_cst) const volatile
	{
		storage_type tmp;
		__asm__ __volatile__ (
			"ld %0, %1\n"
			"cmpd %0, %0\n"
			"bne- 1f\n"
			"1:\n"
			: "=r"(tmp)
			: "m"(v_)
			: "cr0"
		);
		ppc_fence_after(order);
		
		value_type v;
		memcpy(&v, &tmp, sizeof(value_type));
		return v;
	}
	
	value_type
	exchange(value_type v, memory_order order = memory_order_seq_cst) volatile
	{
		storage_type tmp = 0, original;
		memcpy(&tmp, &v, sizeof(value_type));
		ppc_fence_before(order);
		__asm__ (
			"1:\n"
			"ldarx %0,%y1\n"
			"stdcx. %2,%y1\n"
			"bne- 1b\n"
			: "=&b" (original), "+Z"(v_)
			: "b" (tmp)
			: "cr0"
		);
		ppc_fence_after(order);
		memcpy(&v, &original, sizeof(value_type));
		return v;
	}
	
	bool
	compare_exchange_weak(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s, desired_s;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 2f\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	compare_exchange_strong(
		value_type & expected,
		value_type desired,
		memory_order success_order,
		memory_order failure_order) volatile
	{
		storage_type expected_s, desired_s;
		memcpy(&expected_s, &expected, sizeof(value_type));
		memcpy(&desired_s, &desired, sizeof(value_type));
		
		int success;
		ppc_fence_before(success_order);
		__asm__(
			"0: ldarx %0,%y2\n"
			"cmpd %0, %3\n"
			"bne- 2f\n"
			"stdcx. %4,%y2\n"
			"bne- 0b\n"
			"addi %1,0,1\n"
			"1:"
			
			BOOST_ATOMIC_ASM_SLOWPATH_CLEAR
			: "=&b" (expected_s), "=&b" (success), "+Z"(v_)
			: "b" (expected_s), "b" (desired_s)
			: "cr0"
		);
		if (success)
			ppc_fence_after(success_order);
		else
			ppc_fence_after(failure_order);
		memcpy(&expected, &expected_s, sizeof(value_type));
		return success;
	}
	
	bool
	is_lock_free(void) const volatile
	{
		return true;
	}
	
	BOOST_ATOMIC_DECLARE_BASE_OPERATORS
private:
	base_atomic(const base_atomic &) /* = delete */ ;
	void operator=(const base_atomic &) /* = delete */ ;
	storage_type v_;
};
#endif

}
}

#define BOOST_ATOMIC_THREAD_FENCE 2
inline void
atomic_thread_fence(memory_order order)
{
	switch(order) {
		case memory_order_acquire:
			__asm__ __volatile__ ("isync" ::: "memory");
			break;
		case memory_order_release:
#if defined(__powerpc64__)
			__asm__ __volatile__ ("lwsync" ::: "memory");
			break;
#endif
		case memory_order_acq_rel:
		case memory_order_seq_cst:
			__asm__ __volatile__ ("sync" ::: "memory");
		default:;
	}
}

#define BOOST_ATOMIC_SIGNAL_FENCE 2
inline void
atomic_signal_fence(memory_order order)
{
	switch(order) {
		case memory_order_acquire:
		case memory_order_release:
		case memory_order_acq_rel:
		case memory_order_seq_cst:
			__asm__ __volatile__ ("" ::: "memory");
			break;
		default:;
	}
}

}

#endif
